From d134e90a729ae5bb38093882b2a63930bd9e478e Mon Sep 17 00:00:00 2001 From: Lukas Schaefer Date: Wed, 8 Jul 2026 16:41:07 -0400 Subject: [PATCH 1/2] Feat: Add MultimodalChatWithTools and MultimodalContextAgentInteraction Signed-off-by: Lukas Schaefer --- lib/private/TaskProcessing/Manager.php | 4 + .../TaskTypes/MultimodalChatWithTools.php | 135 ++++++++++++++++++ .../MultimodalContextAgentInteraction.php | 126 ++++++++++++++++ 3 files changed, 265 insertions(+) create mode 100644 lib/public/TaskProcessing/TaskTypes/MultimodalChatWithTools.php create mode 100644 lib/public/TaskProcessing/TaskTypes/MultimodalContextAgentInteraction.php diff --git a/lib/private/TaskProcessing/Manager.php b/lib/private/TaskProcessing/Manager.php index 787d65d804ad0..47a2382906f56 100644 --- a/lib/private/TaskProcessing/Manager.php +++ b/lib/private/TaskProcessing/Manager.php @@ -79,6 +79,8 @@ use OCP\TaskProcessing\TaskTypes\ContextWrite; use OCP\TaskProcessing\TaskTypes\GenerateEmoji; use OCP\TaskProcessing\TaskTypes\ImageToTextOpticalCharacterRecognition; +use OCP\TaskProcessing\TaskTypes\MultimodalChatWithTools; +use OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction; use OCP\TaskProcessing\TaskTypes\TextToImage; use OCP\TaskProcessing\TaskTypes\TextToSpeech; use OCP\TaskProcessing\TaskTypes\TextToText; @@ -696,6 +698,7 @@ private function _getTaskTypes(): array { GenerateEmoji::ID => Server::get(GenerateEmoji::class), TextToTextChangeTone::ID => Server::get(TextToTextChangeTone::class), TextToTextChatWithTools::ID => Server::get(TextToTextChatWithTools::class), + MultimodalChatWithTools::ID => Server::get(MultimodalChatWithTools::class), ContextAgentInteraction::ID => Server::get(ContextAgentInteraction::class), TextToTextProofread::ID => Server::get(TextToTextProofread::class), TextToTextReformatParagraphs::ID => Server::get(TextToTextReformatParagraphs::class), @@ -703,6 +706,7 @@ private function _getTaskTypes(): array { AudioToAudioChat::ID => Server::get(AudioToAudioChat::class), AudioToAudioTranslate::ID => Server::get(AudioToAudioTranslate::class), ContextAgentAudioInteraction::ID => Server::get(ContextAgentAudioInteraction::class), + MultimodalContextAgentInteraction::ID => Server::get(MultimodalContextAgentInteraction::class), AnalyzeImages::ID => Server::get(AnalyzeImages::class), ImageToTextOpticalCharacterRecognition::ID => Server::get(ImageToTextOpticalCharacterRecognition::class), ]; diff --git a/lib/public/TaskProcessing/TaskTypes/MultimodalChatWithTools.php b/lib/public/TaskProcessing/TaskTypes/MultimodalChatWithTools.php new file mode 100644 index 0000000000000..c0039795e82ca --- /dev/null +++ b/lib/public/TaskProcessing/TaskTypes/MultimodalChatWithTools.php @@ -0,0 +1,135 @@ +l = $l10nFactory->get('lib'); + } + + /** + * @inheritDoc + * @since 35.0.0 + */ + #[\Override] + public function getName(): string { + // TRANSLATORS Tool calling, also known as function calling, is a structured way to give LLMs the ability to make requests back to the application that called it. You define the tools you want to make available to the model, and the model will make tool requests to your app as necessary to fulfill the prompts you give it. + return $this->l->t('Chat with tools using attachments'); + } + + /** + * @inheritDoc + * @since 35.0.0 + */ + #[\Override] + public function getDescription(): string { + // TRANSLATORS Tool calling, also known as function calling, is a structured way to give LLMs the ability to make requests back to the application that called it. You define the tools you want to make available to the model, and the model will make tool requests to your app as necessary to fulfill the prompts you give it. + return $this->l->t('Chat with the language model with tool calling support and attachments.'); + } + + /** + * @return string + * @since 35.0.0 + */ + #[\Override] + public function getId(): string { + return self::ID; + } + + /** + * @return ShapeDescriptor[] + * @since 35.0.0 + */ + #[\Override] + public function getInputShape(): array { + return [ + 'system_prompt' => new ShapeDescriptor( + $this->l->t('System prompt'), + $this->l->t('Define rules and assumptions that the assistant should follow during the conversation.'), + EShapeType::Text + ), + 'input' => new ShapeDescriptor( + $this->l->t('Chat message'), + $this->l->t('Describe a task that you want the assistant to do or ask a question'), + EShapeType::Text + ), + 'input_attachments' => new ShapeDescriptor( + $this->l->t('Input attachments'), + $this->l->t('Files to send along with the chat message.'), + EShapeType::ListOfFiles + ), + 'tool_message' => new ShapeDescriptor( + $this->l->t('Tool message'), + $this->l->t('The result of tool calls in the last interaction'), + EShapeType::Text + ), + 'history' => new ShapeDescriptor( + $this->l->t('Chat history'), + $this->l->t('The history of chat messages before the current message, starting with a message by the user'), + EShapeType::ListOfTexts + ), + // See https://platform.openai.com/docs/api-reference/chat/create#chat-create-tools for the format + 'tools' => new ShapeDescriptor( + // TRANSLATORS Tool calling, also known as function calling, is a structured way to give LLMs the ability to make requests back to the application that called it. You define the tools you want to make available to the model, and the model will make tool requests to your app as necessary to fulfill the prompts you give it. + $this->l->t('Available tools'), + $this->l->t('The available tools in JSON format'), + EShapeType::Text + ), + ]; + } + + /** + * @return ShapeDescriptor[] + * @since 35.0.0 + */ + #[\Override] + public function getOutputShape(): array { + return [ + 'output' => new ShapeDescriptor( + $this->l->t('Generated response'), + $this->l->t('The response from the chat model'), + EShapeType::Text + ), + 'output_attachments' => new ShapeDescriptor( + $this->l->t('Output attachments'), + $this->l->t('Files generated by the model.'), + EShapeType::ListOfFiles + ), + 'tool_calls' => new ShapeDescriptor( + $this->l->t('Tool calls'), + $this->l->t('Tools call instructions from the model in JSON format'), + EShapeType::Text + ), + ]; + } +} diff --git a/lib/public/TaskProcessing/TaskTypes/MultimodalContextAgentInteraction.php b/lib/public/TaskProcessing/TaskTypes/MultimodalContextAgentInteraction.php new file mode 100644 index 0000000000000..dfc71dc0367a8 --- /dev/null +++ b/lib/public/TaskProcessing/TaskTypes/MultimodalContextAgentInteraction.php @@ -0,0 +1,126 @@ +l = $l10nFactory->get('lib'); + } + + /** + * @inheritDoc + * @since 35.0.0 + */ + #[\Override] + public function getName(): string { + return 'ContextAgent multimodal'; // We do not translate this + } + + /** + * @inheritDoc + * @since 35.0.0 + */ + #[\Override] + public function getDescription(): string { + return $this->l->t('Chat with an agent using attachments'); + } + + /** + * @return string + * @since 35.0.0 + */ + #[\Override] + public function getId(): string { + return self::ID; + } + + /** + * @return ShapeDescriptor[] + * @since 35.0.0 + */ + #[\Override] + public function getInputShape(): array { + return [ + 'input' => new ShapeDescriptor( + $this->l->t('Chat message'), + $this->l->t('A chat message to send to the agent.'), + EShapeType::Text + ), + 'input_attachments' => new ShapeDescriptor( + $this->l->t('Input attachments'), + $this->l->t('Files to send along with the chat message.'), + EShapeType::ListOfFiles + ), + 'confirmation' => new ShapeDescriptor( + $this->l->t('Confirmation'), + $this->l->t('Whether to confirm previously requested actions: 0 for denial and 1 for confirmation.'), + EShapeType::Number + ), + 'conversation_token' => new ShapeDescriptor( + $this->l->t('Conversation token'), + $this->l->t('A token representing the conversation.'), + EShapeType::Text + ), + ]; + } + + /** + * @return ShapeDescriptor[] + * @since 35.0.0 + */ + #[\Override] + public function getOutputShape(): array { + return [ + 'output' => new ShapeDescriptor( + $this->l->t('Generated response'), + $this->l->t('The response from the chat model.'), + EShapeType::Text + ), + 'output_attachments' => new ShapeDescriptor( + $this->l->t('Output attachments'), + $this->l->t('Files generated by the agent.'), + EShapeType::ListOfFiles + ), + 'conversation_token' => new ShapeDescriptor( + $this->l->t('The new conversation token'), + $this->l->t('Send this along with the next interaction.'), + EShapeType::Text + ), + 'actions' => new ShapeDescriptor( + $this->l->t('Requested actions by the agent'), + $this->l->t('Actions that the agent would like to carry out in JSON format.'), + EShapeType::Text + ), + ]; + } +} From 88dc4fd8c35cf750f82243c1dfad1fe6696fcb42 Mon Sep 17 00:00:00 2001 From: Lukas Schaefer Date: Thu, 16 Jul 2026 11:07:19 -0400 Subject: [PATCH 2/2] implement feedback Signed-off-by: Lukas Schaefer --- lib/composer/composer/autoload_classmap.php | 2 ++ lib/composer/composer/autoload_static.php | 2 ++ .../TaskTypes/MultimodalContextAgentInteraction.php | 10 ++++++++++ 3 files changed, 14 insertions(+) diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index c406238f396af..f2b0fc4cec2fb 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -971,6 +971,8 @@ 'OCP\\TaskProcessing\\TaskTypes\\ContextWrite' => $baseDir . '/lib/public/TaskProcessing/TaskTypes/ContextWrite.php', 'OCP\\TaskProcessing\\TaskTypes\\GenerateEmoji' => $baseDir . '/lib/public/TaskProcessing/TaskTypes/GenerateEmoji.php', 'OCP\\TaskProcessing\\TaskTypes\\ImageToTextOpticalCharacterRecognition' => $baseDir . '/lib/public/TaskProcessing/TaskTypes/ImageToTextOpticalCharacterRecognition.php', + 'OCP\\TaskProcessing\\TaskTypes\\MultimodalChatWithTools' => $baseDir . '/lib/public/TaskProcessing/TaskTypes/MultimodalChatWithTools.php', + 'OCP\\TaskProcessing\\TaskTypes\\MultimodalContextAgentInteraction' => $baseDir . '/lib/public/TaskProcessing/TaskTypes/MultimodalContextAgentInteraction.php', 'OCP\\TaskProcessing\\TaskTypes\\TextToImage' => $baseDir . '/lib/public/TaskProcessing/TaskTypes/TextToImage.php', 'OCP\\TaskProcessing\\TaskTypes\\TextToSpeech' => $baseDir . '/lib/public/TaskProcessing/TaskTypes/TextToSpeech.php', 'OCP\\TaskProcessing\\TaskTypes\\TextToText' => $baseDir . '/lib/public/TaskProcessing/TaskTypes/TextToText.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index d04f6e5b986d1..e39c3db6aedac 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -1012,6 +1012,8 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2 'OCP\\TaskProcessing\\TaskTypes\\ContextWrite' => __DIR__ . '/../../..' . '/lib/public/TaskProcessing/TaskTypes/ContextWrite.php', 'OCP\\TaskProcessing\\TaskTypes\\GenerateEmoji' => __DIR__ . '/../../..' . '/lib/public/TaskProcessing/TaskTypes/GenerateEmoji.php', 'OCP\\TaskProcessing\\TaskTypes\\ImageToTextOpticalCharacterRecognition' => __DIR__ . '/../../..' . '/lib/public/TaskProcessing/TaskTypes/ImageToTextOpticalCharacterRecognition.php', + 'OCP\\TaskProcessing\\TaskTypes\\MultimodalChatWithTools' => __DIR__ . '/../../..' . '/lib/public/TaskProcessing/TaskTypes/MultimodalChatWithTools.php', + 'OCP\\TaskProcessing\\TaskTypes\\MultimodalContextAgentInteraction' => __DIR__ . '/../../..' . '/lib/public/TaskProcessing/TaskTypes/MultimodalContextAgentInteraction.php', 'OCP\\TaskProcessing\\TaskTypes\\TextToImage' => __DIR__ . '/../../..' . '/lib/public/TaskProcessing/TaskTypes/TextToImage.php', 'OCP\\TaskProcessing\\TaskTypes\\TextToSpeech' => __DIR__ . '/../../..' . '/lib/public/TaskProcessing/TaskTypes/TextToSpeech.php', 'OCP\\TaskProcessing\\TaskTypes\\TextToText' => __DIR__ . '/../../..' . '/lib/public/TaskProcessing/TaskTypes/TextToText.php', diff --git a/lib/public/TaskProcessing/TaskTypes/MultimodalContextAgentInteraction.php b/lib/public/TaskProcessing/TaskTypes/MultimodalContextAgentInteraction.php index dfc71dc0367a8..fd77189bba816 100644 --- a/lib/public/TaskProcessing/TaskTypes/MultimodalContextAgentInteraction.php +++ b/lib/public/TaskProcessing/TaskTypes/MultimodalContextAgentInteraction.php @@ -91,6 +91,11 @@ public function getInputShape(): array { $this->l->t('A token representing the conversation.'), EShapeType::Text ), + 'memories' => new ShapeDescriptor( + $this->l->t('Memories'), + $this->l->t('Memories to send to the agent.'), + EShapeType::ListOfTexts + ), ]; } @@ -121,6 +126,11 @@ public function getOutputShape(): array { $this->l->t('Actions that the agent would like to carry out in JSON format.'), EShapeType::Text ), + 'sources' => new ShapeDescriptor( + $this->l->t('Sources'), + $this->l->t('Sources of the generated response.'), + EShapeType::ListOfTexts + ), ]; } }