Fix chat request user message serialization

eng/versioning/version_client.txt

@@ -479,6 +479,7 @@ io.clientcore:http-stress;1.0.0-beta.1;1.0.0-beta.1
 # note: The unreleased dependencies will not be manipulated with the automatic PR creation code.
 # In the pom, the version update tag after the version should name the unreleased package and the dependency version:
 # <!-- {x-version-update;unreleased_com.azure:azure-core;dependency} -->
+unreleased_com.azure:azure-core;1.50.0-beta.1
 
 # Released Beta dependencies: Copy the entry from above, prepend "beta_", remove the current
 # version and set the version to the released beta. Released beta dependencies are only valid

sdk/core/azure-core/src/main/java/com/azure/core/util/BinaryData.java

@@ -1608,4 +1608,5 @@ public Mono<BinaryData> toReplayableBinaryDataAsync() {
             return content.toReplayableContentAsync().map(BinaryData::new);
         }
     }
+
 }

sdk/core/azure-core/src/main/java/module-info.java

@@ -44,8 +44,9 @@
     exports com.azure.core.implementation.jackson to com.azure.core.management, com.azure.core.serializer.json.jackson;
 
     // export core utilities to other core packages.
-    exports com.azure.core.implementation.util to com.azure.http.netty, com.azure.core.http.okhttp,
-        com.azure.core.http.jdk.httpclient, com.azure.core.http.vertx, com.azure.core.serializer.json.jackson;
+    exports com.azure.core.implementation.util
+        to com.azure.http.netty, com.azure.core.http.okhttp, com.azure.core.http.jdk.httpclient,
+        com.azure.core.http.vertx, com.azure.core.serializer.json.jackson;
     exports com.azure.core.util.polling.implementation to com.azure.core.experimental;
 
     // exporting some packages specifically for Jackson

sdk/openai/azure-ai-openai/CHANGELOG.md

@@ -8,6 +8,8 @@
 
 ### Bugs Fixed
 
+- Fixed a bug during the serialization and deserialization of the `content` property in the `ChatRequestUserMessage`. ([#39204](https://github.com/Azure/azure-sdk-for-java/pull/39204))
+
 ### Other Changes
 
 ## 1.0.0-beta.9 (2024-06-06)

sdk/openai/azure-ai-openai/README.md

@@ -56,7 +56,7 @@ In order to interact with the Azure OpenAI Service you'll need to create an inst
 Azure OpenAI, provide a valid endpoint URI to an Azure OpenAI resource along with a corresponding key credential,
 token credential, or [Azure Identity][azure_identity] credential that's authorized to use the Azure OpenAI resource. 
 
-#### Create a Azure OpenAI client with key credential
+#### Create an Azure OpenAI client with key credential
 Get Azure OpenAI `key` credential from the Azure Portal.
 
 ```java readme-sample-createSyncClientKeyCredential

sdk/openai/azure-ai-openai/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "java",
   "TagPrefix": "java/openai/azure-ai-openai",
-  "Tag": "java/openai/azure-ai-openai_959f9440df"
+  "Tag": "java/openai/azure-ai-openai_0e2b3dee1d"
 }

sdk/openai/azure-ai-openai/pom.xml

@@ -60,7 +60,7 @@
     <dependency>
       <groupId>com.azure</groupId>
       <artifactId>azure-core</artifactId>
-      <version>1.49.1</version> <!-- {x-version-update;com.azure:azure-core;dependency} -->
+      <version>1.50.0-beta.1</version> <!-- {x-version-update;unreleased_com.azure:azure-core;dependency} -->
     </dependency>
     <dependency>
       <groupId>com.azure</groupId>

sdk/openai/azure-ai-openai/src/main/java/com/azure/ai/openai/OpenAIClient.java

@@ -29,7 +29,6 @@
 import com.azure.core.http.rest.Response;
 import com.azure.core.http.rest.SimpleResponse;
 import com.azure.core.util.BinaryData;
-import com.azure.core.util.logging.ClientLogger;
 import com.azure.ai.openai.implementation.CompletionsUtils;
 import com.azure.ai.openai.implementation.MultipartDataHelper;
 import com.azure.ai.openai.implementation.MultipartDataSerializationResult;
@@ -52,8 +51,6 @@
 @ServiceClient(builder = OpenAIClientBuilder.class)
 public final class OpenAIClient {
 
-    private static final ClientLogger LOGGER = new ClientLogger(OpenAIClient.class);
-
     @Generated
     private final OpenAIClientImpl serviceClient;
 
@@ -1056,7 +1053,7 @@ public Response<String> getAudioTranslationTextWithResponse(String deploymentOrM
      * Gets transcribed text and associated metadata from provided spoken audio data. Audio will be transcribed in the
      * written language corresponding to the language it was spoken in.
      * <p><strong>Response Body Schema</strong></p>
-     * 
+     *
      * <pre>{@code
      * {
      *     text: String (Required)
@@ -1113,7 +1110,7 @@ Response<BinaryData> getAudioTranscriptionAsResponseObjectWithResponse(String de
      * Gets transcribed text and associated metadata from provided spoken audio data. Audio will be transcribed in the
      * written language corresponding to the language it was spoken in.
      * <p><strong>Response Body Schema</strong></p>
-     * 
+     *
      * <pre>{@code
      * String
      * }</pre>
@@ -1141,7 +1138,7 @@ Response<BinaryData> getAudioTranscriptionAsPlainTextWithResponse(String deploym
     /**
      * Gets English language transcribed text and associated metadata from provided spoken audio data.
      * <p><strong>Response Body Schema</strong></p>
-     * 
+     *
      * <pre>{@code
      * {
      *     text: String (Required)
@@ -1191,7 +1188,7 @@ Response<BinaryData> getAudioTranslationAsResponseObjectWithResponse(String depl
     /**
      * Gets English language transcribed text and associated metadata from provided spoken audio data.
      * <p><strong>Response Body Schema</strong></p>
-     * 
+     *
      * <pre>{@code
      * String
      * }</pre>

sdk/openai/azure-ai-openai/src/main/java/com/azure/ai/openai/models/ChatRequestUserMessage.java

@@ -10,9 +10,8 @@
 import com.azure.json.JsonToken;
 import com.azure.json.JsonWriter;
 import java.io.IOException;
-import java.util.List;
-import java.util.stream.Collectors;
 import java.util.Arrays;
+import java.util.List;
 
 /**
  * A request chat message representing user input to the assistant.
@@ -25,6 +24,8 @@ public final class ChatRequestUserMessage extends ChatRequestMessage {
      */
     @Generated
     private final BinaryData content;
+    private final String stringContent;
+    private final List<ChatMessageContentItem> chatMessageContentItems;
 
     /*
      * An optional name for the participant.
@@ -39,6 +40,8 @@ public final class ChatRequestUserMessage extends ChatRequestMessage {
      */
     private ChatRequestUserMessage(BinaryData content) {
         this.content = content;
+        this.chatMessageContentItems = null;
+        this.stringContent = null;
     }
 
     /**
@@ -47,7 +50,9 @@ private ChatRequestUserMessage(BinaryData content) {
      * @param content The plain text content associated with the message.
      */
     public ChatRequestUserMessage(String content) {
-        this(BinaryData.fromString(content));
+        this.content = BinaryData.fromString(content);
+        this.stringContent = content;
+        this.chatMessageContentItems = null;
     }
 
     /**
@@ -56,7 +61,9 @@ public ChatRequestUserMessage(String content) {
      * @param content The collection of structured content associated with the message.
      */
     public ChatRequestUserMessage(ChatMessageContentItem[] content) {
-        this(BinaryData.fromObject(Arrays.stream(content).map(BinaryData::fromObject).collect(Collectors.toList())));
+        this.content = BinaryData.fromObject(content);
+        this.chatMessageContentItems = Arrays.asList(content);
+        this.stringContent = null;
     }
 
     /**
@@ -65,7 +72,9 @@ public ChatRequestUserMessage(ChatMessageContentItem[] content) {
      * @param content The collection of structured content associated with the message.
      */
     public ChatRequestUserMessage(List<ChatMessageContentItem> content) {
-        this(BinaryData.fromObject(content.stream().map(BinaryData::fromObject).collect(Collectors.toList())));
+        this.content = BinaryData.fromObject(content);
+        this.stringContent = null;
+        this.chatMessageContentItems = content;
     }
 
     /**
@@ -123,7 +132,12 @@ public ChatRole getRole() {
     @Override
     public JsonWriter toJson(JsonWriter jsonWriter) throws IOException {
         jsonWriter.writeStartObject();
-        jsonWriter.writeBinaryField("content", this.content.toBytes());
+        if (stringContent != null) {
+            jsonWriter.writeStringField("content", stringContent);
+        } else if (chatMessageContentItems != null) {
+            jsonWriter.writeArrayField("content", chatMessageContentItems, JsonWriter::writeJson);
+        }
+
         jsonWriter.writeStringField("role", this.role == null ? null : this.role.toString());
         jsonWriter.writeStringField("name", this.name);
         return jsonWriter.writeEndObject();
@@ -138,7 +152,6 @@ public JsonWriter toJson(JsonWriter jsonWriter) throws IOException {
      * @throws IllegalStateException If the deserialized JSON object was missing any required properties.
      * @throws IOException If an error occurs while reading the ChatRequestUserMessage.
      */
-    @Generated
     public static ChatRequestUserMessage fromJson(JsonReader jsonReader) throws IOException {
         return jsonReader.readObject(reader -> {
             BinaryData content = null;
@@ -148,7 +161,17 @@ public static ChatRequestUserMessage fromJson(JsonReader jsonReader) throws IOEx
                 String fieldName = reader.getFieldName();
                 reader.nextToken();
                 if ("content".equals(fieldName)) {
-                    content = reader.getNullable(nonNullReader -> BinaryData.fromObject(nonNullReader.readUntyped()));
+                    if (reader.currentToken() == JsonToken.STRING) {
+                        content = BinaryData.fromString(reader.getString());
+                    } else if (reader.currentToken() == JsonToken.START_ARRAY) {
+                        content = BinaryData.fromObject(reader.readArray(arrayReader -> arrayReader
+                            .readObject(ChatMessageContentItem::fromJson)));
+                    } else if (reader.currentToken() == JsonToken.NULL) {
+                        content = null;
+                    } else {
+                        throw new IllegalStateException("Unexpected 'content' type found when deserializing"
+                            + " ChatRequestUserMessage JSON object: " + reader.currentToken());
+                    }
                 } else if ("role".equals(fieldName)) {
                     role = ChatRole.fromString(reader.getString());
                 } else if ("name".equals(fieldName)) {

sdk/openai/azure-ai-openai/src/test/java/com/azure/ai/openai/models/ChatRequestMessageUnitTests.java

@@ -3,9 +3,14 @@
 
 package com.azure.ai.openai.models;
 
+import com.azure.core.util.BinaryData;
 import org.junit.jupiter.api.Test;
 
+import java.util.Arrays;
+
+import static com.azure.ai.openai.models.ChatRole.USER;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
 
 /**
  * Unit tests for ChatRequestMessage.
@@ -14,33 +19,125 @@ public class ChatRequestMessageUnitTests {
     String content = "You are a helpful assistant.";
     String name = "Alice";
     @Test
-    public void testChatRequestUserMessage() {
+    public void testChatRequestUserMessageContentInString() {
+        // String type content
         ChatRequestUserMessage chatRequestUserMessage = new ChatRequestUserMessage(content).setName(name);
-
         assertEquals(content, chatRequestUserMessage.getContent().toString());
+        assertEquals(name, chatRequestUserMessage.getName());
+        assertEquals(USER, chatRequestUserMessage.getRole());
+
+        // Test it in ChatCompletionsOptions
+        ChatCompletionsOptions chatCompletionsOptions = new ChatCompletionsOptions(Arrays.asList(chatRequestUserMessage));
+        assertChatCompletionsOptions(chatCompletionsOptions);
+    }
+
+    @Test
+    public void testUserMessageContentSameStructureAfterConverted() {
+        final ChatRequestUserMessage user = new ChatRequestUserMessage(Arrays.asList(
+            new ChatMessageTextContentItem("textContent"),
+            new ChatMessageImageContentItem(new ChatMessageImageUrl("testImage"))
+        ));
+        final String userMessageInString = BinaryData.fromObject(user).toString();
+        final ChatRequestUserMessage convertedUserMessageFromString = BinaryData.fromString(userMessageInString).toObject(ChatRequestUserMessage.class);
+        assertEquals(user.getContent().toString(), convertedUserMessageFromString.getContent().toString());
+    }
+
+    @Test
+    public void testChatRequestUserMessageContentInChatMessageContentItemList() {
+        ChatRequestUserMessage userMessage = new ChatRequestUserMessage(new ChatMessageContentItem[] {
+            new ChatMessageTextContentItem("textContent"),
+            new ChatMessageImageContentItem(new ChatMessageImageUrl("testImage"))
+        });
+        assertChatRequestUserMessage(userMessage);
+    }
+
+    @Test
+    public void testChatRequestUserMessageContentInChatMessageContentItemArray() {
+        ChatRequestUserMessage userMessage = new ChatRequestUserMessage(Arrays.asList(
+            new ChatMessageTextContentItem("textContent"),
+            new ChatMessageImageContentItem(new ChatMessageImageUrl("testImage"))
+        ));
+        assertChatRequestUserMessage(userMessage);
     }
 
     @Test
     public void testChatRequestSystemMessage() {
         ChatRequestSystemMessage chatRequestSystemMessage = new ChatRequestSystemMessage(content).setName(name);
         assertEquals(content, chatRequestSystemMessage.getContent());
+
+        // Test it in ChatCompletionsOptions
+        ChatCompletionsOptions chatCompletionsOptions = new ChatCompletionsOptions(Arrays.asList(chatRequestSystemMessage));
+        assertChatCompletionsOptions(chatCompletionsOptions);
     }
 
     @Test
     public void testChatRequestAssistantMessage() {
         ChatRequestAssistantMessage chatRequestAssistantMessage = new ChatRequestAssistantMessage(content).setName(name);
         assertEquals(content, chatRequestAssistantMessage.getContent());
+
+        // Test it in ChatCompletionsOptions
+        ChatCompletionsOptions chatCompletionsOptions = new ChatCompletionsOptions(Arrays.asList(chatRequestAssistantMessage));
+        assertChatCompletionsOptions(chatCompletionsOptions);
     }
 
     @Test
     public void testChatRequestToolMessage() {
         ChatRequestToolMessage chatRequestToolMessage = new ChatRequestToolMessage(content, "tool_call_id_value");
         assertEquals(content, chatRequestToolMessage.getContent());
+
+        // Test it in ChatCompletionsOptions
+        ChatCompletionsOptions chatCompletionsOptions = new ChatCompletionsOptions(Arrays.asList(chatRequestToolMessage));
+        assertChatCompletionsOptions(chatCompletionsOptions);
     }
 
     @Test
     public void testChatRequestFunctionMessage() {
         ChatRequestFunctionMessage chatRequestFunctionMessage = new ChatRequestFunctionMessage(name, content);
         assertEquals(content, chatRequestFunctionMessage.getContent());
+
+        // Test it in ChatCompletionsOptions
+        ChatCompletionsOptions chatCompletionsOptions = new ChatCompletionsOptions(Arrays.asList(chatRequestFunctionMessage));
+        assertChatCompletionsOptions(chatCompletionsOptions);
+    }
+
+    private void assertChatRequestUserMessage(ChatRequestUserMessage userMessage) {
+        String userMessageInString = BinaryData.fromObject(userMessage).toString();
+        ChatRequestUserMessage converted = BinaryData.fromString(userMessageInString)
+                .toObject(ChatRequestUserMessage.class);
+        assertEquals(userMessage.getContent().toString(), converted.getContent().toString());
+        assertEquals(userMessage.getName(), converted.getName());
+        assertEquals(USER, converted.getRole());
+    }
+
+    private void assertChatCompletionsOptions(ChatCompletionsOptions chatCompletionsOptions) {
+        String chatCompletionsOptionsInString = BinaryData.fromObject(chatCompletionsOptions).toString();
+        ChatCompletionsOptions converted = BinaryData.fromString(chatCompletionsOptionsInString)
+                .toObject(ChatCompletionsOptions.class);
+
+        converted.getMessages().forEach(message -> {
+            if (message instanceof ChatRequestUserMessage) {
+                ChatRequestUserMessage userMessage = (ChatRequestUserMessage) message;
+                ChatRequestUserMessage expectedUserMessage = (ChatRequestUserMessage) chatCompletionsOptions.getMessages().get(0);
+                assertEquals(expectedUserMessage.getContent().toString(), userMessage.getContent().toString());
+            } else if (message instanceof ChatRequestSystemMessage) {
+                ChatRequestSystemMessage systemMessage = (ChatRequestSystemMessage) message;
+                ChatRequestSystemMessage expectedSystemMessage = (ChatRequestSystemMessage) chatCompletionsOptions.getMessages().get(0);
+                assertEquals(expectedSystemMessage.getContent(), systemMessage.getContent());
+            } else if (message instanceof ChatRequestAssistantMessage) {
+                ChatRequestAssistantMessage assistantMessage = (ChatRequestAssistantMessage) message;
+                ChatRequestAssistantMessage expectedAssistantMessage = (ChatRequestAssistantMessage) chatCompletionsOptions.getMessages().get(0);
+                assertEquals(expectedAssistantMessage.getContent(), assistantMessage.getContent());
+            } else if (message instanceof ChatRequestToolMessage) {
+                ChatRequestToolMessage toolMessage = (ChatRequestToolMessage) message;
+                ChatRequestToolMessage expectedToolMessage = (ChatRequestToolMessage) chatCompletionsOptions.getMessages().get(0);
+                assertEquals(expectedToolMessage.getContent(), toolMessage.getContent());
+            } else if (message instanceof ChatRequestFunctionMessage) {
+                ChatRequestFunctionMessage functionMessage = (ChatRequestFunctionMessage) message;
+                ChatRequestFunctionMessage expectedFunctionMessage = (ChatRequestFunctionMessage) chatCompletionsOptions.getMessages().get(0);
+                assertEquals(expectedFunctionMessage.getContent(), functionMessage.getContent());
+            } else {
+                assertFalse(true, "Unexpected message type");
+            }
+        });
     }
 }