From b61f0f2122bd21648fe0ed47ec42e4d37b473298 Mon Sep 17 00:00:00 2001 From: Caleb Areeveso Date: Wed, 8 Jul 2026 20:22:57 +0000 Subject: [PATCH 01/16] agent: Add image generation appfunction Change-Id: Ie5a3a97c4dc2eefe349f0df22313f2bc12fcfe01 --- agent/app/build.gradle.kts | 7 +- agent/app/src/main/AndroidManifest.xml | 10 + .../agent/data/BuiltInAppFunctions.kt | 181 +++++- .../agent/data/GeminiProviderImpl.kt | 369 +++++------ .../appfunctions/agent/data/db/AppDatabase.kt | 5 +- .../agent/data/db/entities/MessageEntity.kt | 52 +- .../appfunctions/agent/di/DataModule.kt | 22 +- .../agent/domain/AgentOrchestrator.kt | 586 +++++++++--------- .../agent/domain/chat/SendMessageUseCase.kt | 67 +- .../ui/screens/agentdemo/AgentDemoScreen.kt | 361 ++++++----- agent/app/src/main/res/xml/file_paths.xml | 5 + .../MessageAttachmentConverterTest.kt | 56 ++ .../agent/domain/AgentOrchestratorTest.kt | 326 ++++++---- agent/gradle/libs.versions.toml | 1 + 14 files changed, 1229 insertions(+), 819 deletions(-) create mode 100644 agent/app/src/main/res/xml/file_paths.xml create mode 100644 agent/app/src/test/java/com/example/appfunctions/agent/data/db/entities/MessageAttachmentConverterTest.kt diff --git a/agent/app/build.gradle.kts b/agent/app/build.gradle.kts index 3a63ee6..d31e3a6 100644 --- a/agent/app/build.gradle.kts +++ b/agent/app/build.gradle.kts @@ -16,6 +16,7 @@ plugins { alias(libs.plugins.android.application) alias(libs.plugins.kotlin.compose) + alias(libs.plugins.kotlin.serialization) alias(libs.plugins.ksp) alias(libs.plugins.hilt) alias(libs.plugins.screenshot) @@ -60,11 +61,13 @@ android { dimension = "mode" buildConfigField("Boolean", "IS_RETAIL", "true") - val containsRetail = gradle.startParameter.taskNames.any { it.contains("Retail", ignoreCase = true) } + val containsRetail = gradle.startParameter.taskNames.any { + it.contains("Retail", ignoreCase = true) + } val apiKey = project.findProperty("GEMINI_API_KEY") as? String ?: "" if (containsRetail && apiKey.isEmpty()) { throw GradleException( - "GEMINI_API_KEY project property is required for retail builds. Pass it using -PGEMINI_API_KEY=your_key", + "GEMINI_API_KEY project property is required for retail builds. Pass it using -PGEMINI_API_KEY=your_key" ) } buildConfigField("String", "GEMINI_API_KEY", "\"$apiKey\"") diff --git a/agent/app/src/main/AndroidManifest.xml b/agent/app/src/main/AndroidManifest.xml index 8c4a97e..ea05688 100644 --- a/agent/app/src/main/AndroidManifest.xml +++ b/agent/app/src/main/AndroidManifest.xml @@ -57,6 +57,16 @@ android:theme="@style/Theme.AppCompat.DayNight.DarkActionBar" android:exported="false" tools:replace="android:label,android:theme" /> + + + + + os.write(requestJson.toString().toByteArray(Charsets.UTF_8)) + } + + val responseCode = connection.responseCode + if (responseCode != HttpURLConnection.HTTP_OK) { + val errorBody = + connection.errorStream?.bufferedReader()?.use { it.readText() } + ?: "HTTP $responseCode" + throw IllegalStateException( + "Image generation failed ($responseCode): $errorBody" + ) + } + + val responseText = connection.inputStream.bufferedReader().use { it.readText() } + val responseJson = JSONObject(responseText) + val candidates = responseJson.optJSONArray("candidates") + if (candidates == null || candidates.length() == 0) { + throw IllegalStateException( + "No candidates returned from Gemini image generation API" + ) + } + + val parts = + candidates + .getJSONObject(0) + .optJSONObject("content") + ?.optJSONArray("parts") + if (parts == null || parts.length() == 0) { + throw IllegalStateException("No parts returned in candidate content") + } + + var base64Data: String? = null + var mimeType = "image/png" + for (i in 0 until parts.length()) { + val part = parts.getJSONObject(i) + val inlineData = + part.optJSONObject("inlineData") + ?: part.optJSONObject("inline_data") + if (inlineData != null) { + base64Data = inlineData.optString("data") + val returnedMime = + inlineData + .optString("mimeType") + .takeIf { it.isNotBlank() } + ?: inlineData.optString("mime_type") + if (returnedMime.isNotBlank()) { + mimeType = returnedMime + } + break + } + } + + if (base64Data.isNullOrBlank()) { + throw IllegalStateException("No inlineData image found in response parts") + } + + val imageBytes = Base64.decode(base64Data, Base64.DEFAULT) + val extension = + if (mimeType.contains("jpeg") || mimeType.contains("jpg")) "jpg" else "png" + val cachedFile = + File( + context.cacheDir, + "generated_${UUID.randomUUID()}.$extension" + ) + cachedFile.writeBytes(imageBytes) + + val authority = "${context.packageName}.fileprovider" + val contentUri = FileProvider.getUriForFile(context, authority, cachedFile) + GeneratedImageResult( + imageUri = contentUri.toString(), + mimeType = mimeType, + prompt = prompt + ) + } finally { + connection.disconnect() + } + } + + /** Represents the result of an image generation request. */ + @AppFunctionSerializable + data class GeneratedImageResult( + /** The remote URI or URL of the generated image. */ + val imageUri: String, + /** The MIME type of the generated image. */ + val mimeType: String, + /** The original prompt used to generate the image. */ + val prompt: String ) } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt index 7e2bd18..e061216 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt @@ -29,6 +29,9 @@ import io.ktor.client.statement.HttpResponse import io.ktor.client.statement.bodyAsText import io.ktor.http.ContentType import io.ktor.http.contentType +import java.time.LocalDate +import javax.inject.Inject +import javax.inject.Singleton import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonArray import kotlinx.serialization.json.JsonElement @@ -39,216 +42,220 @@ import kotlinx.serialization.json.jsonArray import kotlinx.serialization.json.jsonObject import kotlinx.serialization.json.jsonPrimitive import kotlinx.serialization.json.put -import java.time.LocalDate -import javax.inject.Inject -import javax.inject.Singleton @Singleton class GeminiProviderImpl - @Inject - constructor( - private val httpClient: HttpClient, - private val toolConverter: GeminiToolConverter, - ) : LlmProvider { - override suspend fun generateResponse( - previousInteractionId: String?, - input: LlmInput, - tools: List, - apiKey: String, - modelName: String, - ): LlmResponse { - val convertedTools = - tools.mapNotNull { tool -> - try { - buildJsonObject { - put(KEY_TYPE, JsonPrimitive(VALUE_FUNCTION)) - val functionSchema = toolConverter.convert(tool) - functionSchema.forEach { (key, value) -> put(key, value) } - } - } catch (e: IllegalArgumentException) { - Log.e(TAG, "Failed to convert tool ${tool.id}: ${e.message}", e) - null +@Inject +constructor( + private val httpClient: HttpClient, + private val toolConverter: GeminiToolConverter +) : LlmProvider { + override suspend fun generateResponse( + previousInteractionId: String?, + input: LlmInput, + tools: List, + apiKey: String, + modelName: String + ): LlmResponse { + val convertedTools = + tools.mapNotNull { tool -> + try { + buildJsonObject { + put(KEY_TYPE, JsonPrimitive(VALUE_FUNCTION)) + val functionSchema = toolConverter.convert(tool) + functionSchema.forEach { (key, value) -> put(key, value) } } + } catch (e: IllegalArgumentException) { + Log.e(TAG, "Failed to convert tool ${tool.id}: ${e.message}", e) + null } + } - val requestBody = - buildJsonObject { - put(KEY_MODEL, JsonPrimitive(modelName)) - - put(KEY_SYSTEM_INSTRUCTION, JsonPrimitive(getSystemInstruction())) - - when (input) { - is LlmInput.ToolResponse -> { - val inputElement = - kotlinx.serialization.json.buildJsonArray { - input.outputs.forEach { output -> - val matchingTool = tools.find { it.id == output.functionId } - val mappedName = - if (matchingTool != null) { - toolConverter.getToolName(matchingTool) - } else { - output.functionId - } + val requestBody = + buildJsonObject { + put(KEY_MODEL, JsonPrimitive(modelName)) - add( - buildJsonObject { - put("type", "function_result") - put("name", mappedName) - if (output.callId.isNotEmpty()) { - put("call_id", output.callId) - } - put("result", output.result) - }, - ) - } - } - put(KEY_INPUT, inputElement) - } - is LlmInput.UserMessage -> { - put(KEY_INPUT, JsonPrimitive(input.text)) - } - } + put(KEY_SYSTEM_INSTRUCTION, JsonPrimitive(getSystemInstruction())) - if (convertedTools.isNotEmpty()) { - put(KEY_TOOLS, kotlinx.serialization.json.JsonArray(convertedTools)) - } + when (input) { + is LlmInput.ToolResponse -> { + val inputElement = + kotlinx.serialization.json.buildJsonArray { + input.outputs.forEach { output -> + val matchingTool = tools.find { it.id == output.functionId } + val mappedName = + if (matchingTool != null) { + toolConverter.getToolName(matchingTool) + } else { + output.functionId + } - if (previousInteractionId != null) { - put(KEY_PREVIOUS_INTERACTION_ID, JsonPrimitive(previousInteractionId)) + add( + buildJsonObject { + put("type", "function_result") + put("name", mappedName) + if (output.callId.isNotEmpty()) { + put("call_id", output.callId) + } + put("result", output.result) + } + ) + } + } + put(KEY_INPUT, inputElement) + } + is LlmInput.UserMessage -> { + put(KEY_INPUT, JsonPrimitive(input.text)) } } - Log.d(TAG, "Gemini Request Body: $requestBody") + if (convertedTools.isNotEmpty()) { + put(KEY_TOOLS, kotlinx.serialization.json.JsonArray(convertedTools)) + } - val response: HttpResponse = - try { - httpClient.post(GEMINI_INTERACTIONS_URL) { - contentType(ContentType.Application.Json) - header(HEADER_API_KEY, apiKey) - header(HEADER_API_REVISION, VALUE_API_REVISION) - setBody(requestBody) - } - } catch (e: Exception) { - return LlmResponse.Error("Network error: ${e.message}") + if (previousInteractionId != null) { + put(KEY_PREVIOUS_INTERACTION_ID, JsonPrimitive(previousInteractionId)) } + } - val responseBodyText = response.bodyAsText() - Log.d(TAG, "Gemini Response Body: $responseBodyText") + Log.d(TAG, "Gemini Request Body: $requestBody") - if (response.status.value !in 200..299) { - return LlmResponse.Error("Gemini API error: ${response.status} - $responseBodyText") + val response: HttpResponse = + try { + httpClient.post(GEMINI_INTERACTIONS_URL) { + contentType(ContentType.Application.Json) + header(HEADER_API_KEY, apiKey) + header(HEADER_API_REVISION, VALUE_API_REVISION) + setBody(requestBody) + } + } catch (e: Exception) { + return LlmResponse.Error("Network error: ${e.message}") } - val jsonResponse = Json.parseToJsonElement(responseBodyText).jsonObject - - val newInteractionId = - jsonResponse[KEY_ID]?.jsonPrimitive?.content - ?: return LlmResponse.Error("Missing interaction ID in response") - val steps = jsonResponse[KEY_STEPS]?.jsonArray - - val parts = mutableListOf() - - if (steps != null) { - for (step in steps) { - val stepObj = step.jsonObject - val stepType = stepObj[KEY_TYPE]?.jsonPrimitive?.content - - if (stepType == VALUE_MODEL_OUTPUT) { - val content = stepObj[KEY_CONTENT]?.jsonArray - if (content != null) { - for (part in content) { - val partObj = part.jsonObject - val partType = partObj[KEY_TYPE]?.jsonPrimitive?.content - if (partType == VALUE_TEXT) { - val text = partObj[KEY_TEXT]?.jsonPrimitive?.content - if (text != null) { - parts.add(LlmResponsePart.Text(text)) - } + val responseBodyText = response.bodyAsText() + Log.d(TAG, "Gemini Response Body: $responseBodyText") + + if (response.status.value !in 200..299) { + return LlmResponse.Error("Gemini API error: ${response.status} - $responseBodyText") + } + + val jsonResponse = Json.parseToJsonElement(responseBodyText).jsonObject + + val newInteractionId = + jsonResponse[KEY_ID]?.jsonPrimitive?.content + ?: return LlmResponse.Error("Missing interaction ID in response") + val steps = jsonResponse[KEY_STEPS]?.jsonArray + + val parts = mutableListOf() + + if (steps != null) { + for (step in steps) { + val stepObj = step.jsonObject + val stepType = stepObj[KEY_TYPE]?.jsonPrimitive?.content + + if (stepType == VALUE_MODEL_OUTPUT) { + val content = stepObj[KEY_CONTENT]?.jsonArray + if (content != null) { + for (part in content) { + val partObj = part.jsonObject + val partType = partObj[KEY_TYPE]?.jsonPrimitive?.content + if (partType == VALUE_TEXT) { + val text = partObj[KEY_TEXT]?.jsonPrimitive?.content + if (text != null) { + parts.add(LlmResponsePart.Text(text)) } } } - } else if (stepType == VALUE_FUNCTION_CALL) { - val name = - stepObj[KEY_NAME]?.jsonPrimitive?.content - ?: return LlmResponse.Error("Called function without a name") - val matchingTool = - tools.find { tool -> toolConverter.getToolName(tool) == name } - ?: return LlmResponse.Error("Called unknown function: $name") - - val packageName = matchingTool.packageName - val functionId = matchingTool.id - - val args = stepObj[KEY_ARGUMENTS]?.jsonObject - val argumentsMap = mutableMapOf() - if (args != null) { - for ((key, value) in args) { - argumentsMap[key] = value.toPrimitive() - } + } + } else if (stepType == VALUE_FUNCTION_CALL) { + val name = + stepObj[KEY_NAME]?.jsonPrimitive?.content + ?: return LlmResponse.Error("Called function without a name") + val matchingTool = + tools.find { tool -> toolConverter.getToolName(tool) == name } + ?: return LlmResponse.Error("Called unknown function: $name") + + val packageName = matchingTool.packageName + val functionId = matchingTool.id + + val args = stepObj[KEY_ARGUMENTS]?.jsonObject + val argumentsMap = mutableMapOf() + if (args != null) { + for ((key, value) in args) { + argumentsMap[key] = value.toPrimitive() } - val callId = - stepObj["id"]?.jsonPrimitive?.content - ?: return LlmResponse.Error("Function call missing call_id in response") - parts.add( - LlmResponsePart.ToolCall( - packageName = packageName, - functionId = functionId, - arguments = argumentsMap, - callId = callId, - ), - ) - } else { - Log.d(TAG, "Unsupported step type: $stepType") } + val callId = + stepObj["id"]?.jsonPrimitive?.content + ?: return LlmResponse.Error( + "Function call missing call_id in response" + ) + parts.add( + LlmResponsePart.ToolCall( + packageName = packageName, + functionId = functionId, + arguments = argumentsMap, + callId = callId + ) + ) + } else { + Log.d(TAG, "Unsupported step type: $stepType") } } - - return LlmResponse.Success( - interactionId = newInteractionId, - parts = parts, - ) } - private fun JsonElement.toPrimitive(): Any? { - return when (this) { - is JsonPrimitive -> { - if (this.isString) return this.content - return this.content.toBooleanStrictOrNull() - ?: this.content.toLongOrNull() - ?: this.content.toDoubleOrNull() - } - is JsonObject -> this.mapValues { it.value.toPrimitive() } - is JsonArray -> this.map { it.toPrimitive() } + return LlmResponse.Success( + interactionId = newInteractionId, + parts = parts + ) + } + + private fun JsonElement.toPrimitive(): Any? { + return when (this) { + is JsonPrimitive -> { + if (this.isString) return this.content + return this.content.toBooleanStrictOrNull() + ?: this.content.toLongOrNull() + ?: this.content.toDoubleOrNull() } + is JsonObject -> this.mapValues { it.value.toPrimitive() } + is JsonArray -> this.map { it.toPrimitive() } } + } - private fun getSystemInstruction(): String { - val currentDate = LocalDate.now().toString() - return "You are an assistant running on Android. Be concise, direct and helpful. Today's date is $currentDate." - } + private fun getSystemInstruction(): String { + val currentDate = LocalDate.now().toString() + return """ + You are an AI assistant running on Android. Today's date is $currentDate. + Always reply to the user using concise, friendly, natural human language. Never respond to the user with raw JSON or structured code blocks unless explicitly asked to write code. + When a user asks you to generate an image, call the generateImage tool. After generateImage completes, confirm to the user in a natural sentence (for example: "I have generated that image for you."). + When a user asks you to generate an image and use it in an app (for example, setting a chat wallpaper or attaching an image to a note), first call generateImage, then call the target app function passing the returned imageUri. + """.trimIndent() + } - companion object { - private const val GEMINI_INTERACTIONS_URL = - "https://generativelanguage.googleapis.com/v1beta/interactions" - private const val TAG = "GeminiProvider" - - private const val KEY_MODEL = "model" - private const val KEY_INPUT = "input" - private const val KEY_TOOLS = "tools" - private const val KEY_TYPE = "type" - private const val VALUE_FUNCTION = "function" - private const val KEY_PREVIOUS_INTERACTION_ID = "previous_interaction_id" - private const val KEY_SYSTEM_INSTRUCTION = "system_instruction" - private const val HEADER_API_KEY = "x-goog-api-key" - private const val KEY_ID = "id" - private const val KEY_STEPS = "steps" - private const val KEY_CONTENT = "content" - private const val VALUE_MODEL_OUTPUT = "model_output" - private const val VALUE_TEXT = "text" - private const val KEY_TEXT = "text" - private const val VALUE_FUNCTION_CALL = "function_call" - private const val KEY_NAME = "name" - private const val KEY_ARGUMENTS = "arguments" - private const val HEADER_API_REVISION = "Api-Revision" - private const val VALUE_API_REVISION = "2026-05-20" - } + companion object { + private const val GEMINI_INTERACTIONS_URL = + "https://generativelanguage.googleapis.com/v1beta/interactions" + private const val TAG = "GeminiProvider" + + private const val KEY_MODEL = "model" + private const val KEY_INPUT = "input" + private const val KEY_TOOLS = "tools" + private const val KEY_TYPE = "type" + private const val VALUE_FUNCTION = "function" + private const val KEY_PREVIOUS_INTERACTION_ID = "previous_interaction_id" + private const val KEY_SYSTEM_INSTRUCTION = "system_instruction" + private const val HEADER_API_KEY = "x-goog-api-key" + private const val KEY_ID = "id" + private const val KEY_STEPS = "steps" + private const val KEY_CONTENT = "content" + private const val VALUE_MODEL_OUTPUT = "model_output" + private const val VALUE_TEXT = "text" + private const val KEY_TEXT = "text" + private const val VALUE_FUNCTION_CALL = "function_call" + private const val KEY_NAME = "name" + private const val KEY_ARGUMENTS = "arguments" + private const val HEADER_API_REVISION = "Api-Revision" + private const val VALUE_API_REVISION = "2026-05-20" } +} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/db/AppDatabase.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/db/AppDatabase.kt index ae383e6..46d31c1 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/db/AppDatabase.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/db/AppDatabase.kt @@ -17,11 +17,14 @@ package com.example.appfunctions.agent.data.db import androidx.room.Database import androidx.room.RoomDatabase +import androidx.room.TypeConverters import com.example.appfunctions.agent.data.db.dao.ChatDao +import com.example.appfunctions.agent.data.db.entities.MessageAttachmentConverter import com.example.appfunctions.agent.data.db.entities.MessageEntity import com.example.appfunctions.agent.data.db.entities.ThreadEntity -@Database(entities = [ThreadEntity::class, MessageEntity::class], version = 2, exportSchema = false) +@Database(entities = [ThreadEntity::class, MessageEntity::class], version = 3, exportSchema = false) +@TypeConverters(MessageAttachmentConverter::class) abstract class AppDatabase : RoomDatabase() { abstract fun chatDao(): ChatDao } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/db/entities/MessageEntity.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/db/entities/MessageEntity.kt index 075700b..7144b03 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/db/entities/MessageEntity.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/db/entities/MessageEntity.kt @@ -19,19 +19,22 @@ import androidx.room.Entity import androidx.room.ForeignKey import androidx.room.Index import androidx.room.PrimaryKey +import kotlinx.serialization.Serializable +import kotlinx.serialization.encodeToString +import kotlinx.serialization.json.Json @Entity( tableName = "messages", foreignKeys = - [ - ForeignKey( - entity = ThreadEntity::class, - parentColumns = ["threadId"], - childColumns = ["threadId"], - onDelete = ForeignKey.CASCADE, - ), - ], - indices = [Index("threadId")], + [ + ForeignKey( + entity = ThreadEntity::class, + parentColumns = ["threadId"], + childColumns = ["threadId"], + onDelete = ForeignKey.CASCADE + ) + ], + indices = [Index("threadId")] ) data class MessageEntity( @PrimaryKey val messageId: String, @@ -46,15 +49,42 @@ data class MessageEntity( */ val pendingIntentId: String? = null, val targetPackageName: String? = null, + val attachments: List = emptyList() ) +@Serializable +data class MessageAttachment( + val uri: String, + val mimeType: String +) + +class MessageAttachmentConverter { + private val dbJson = Json { + ignoreUnknownKeys = true + encodeDefaults = true + } + + @androidx.room.TypeConverter + fun fromAttachments(attachments: List): String = + dbJson.encodeToString(attachments) + + @androidx.room.TypeConverter + fun toAttachments(jsonString: String?): List = + if (jsonString.isNullOrBlank()) { + emptyList() + } else { + runCatching { dbJson.decodeFromString>(jsonString) } + .getOrDefault(emptyList()) + } +} + enum class MessageRole { USER, - ASSISTANT, + ASSISTANT } enum class MessageProcessingStatus { PENDING_AGENT_RESPONSE, PROCESSED, - FAILED, + FAILED } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt b/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt index 78ddcc7..bdc1410 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt @@ -39,10 +39,10 @@ import io.ktor.client.engine.okhttp.OkHttp import io.ktor.client.plugins.HttpTimeout import io.ktor.client.plugins.contentnegotiation.ContentNegotiation import io.ktor.serialization.kotlinx.json.json -import kotlinx.serialization.json.Json import javax.inject.Singleton +import kotlinx.serialization.json.Json -private val Context.settingsDataStore: DataStore by +val Context.settingsDataStore: DataStore by preferencesDataStore(name = "settings") @Module @@ -58,32 +58,30 @@ abstract class DataModule { @Binds @Singleton - abstract fun bindPendingIntentRepository(impl: InMemoryPendingIntentRepository): PendingIntentRepository + abstract fun bindPendingIntentRepository( + impl: InMemoryPendingIntentRepository + ): PendingIntentRepository @Binds @Singleton abstract fun bindLlmProviderFactory( - impl: com.example.appfunctions.agent.data.LlmProviderFactoryImpl, + impl: com.example.appfunctions.agent.data.LlmProviderFactoryImpl ): com.example.appfunctions.agent.domain.LlmProviderFactory companion object { @Provides @Singleton - fun provideDataStore( - @ApplicationContext context: Context, - ): DataStore { + fun provideDataStore(@ApplicationContext context: Context): DataStore { return context.settingsDataStore } @Provides @Singleton - fun provideAppDatabase( - @ApplicationContext context: Context, - ): AppDatabase { + fun provideAppDatabase(@ApplicationContext context: Context): AppDatabase { return Room.databaseBuilder( context, AppDatabase::class.java, - "app_database", + "app_database" ) .fallbackToDestructiveMigration() .build() @@ -105,7 +103,7 @@ abstract class DataModule { ignoreUnknownKeys = true prettyPrint = true isLenient = true - }, + } ) } install(HttpTimeout) { socketTimeoutMillis = 30000 } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt index b48931d..a426cdc 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt @@ -19,6 +19,7 @@ import android.util.Log import androidx.appfunctions.metadata.AppFunctionMetadata import com.example.appfunctions.agent.data.LlmProviderName import com.example.appfunctions.agent.data.SettingsRepository +import com.example.appfunctions.agent.data.db.entities.MessageAttachment import com.example.appfunctions.agent.data.db.entities.MessageEntity import com.example.appfunctions.agent.data.db.entities.MessageProcessingStatus import com.example.appfunctions.agent.data.db.entities.MessageRole @@ -35,6 +36,8 @@ import com.example.appfunctions.agent.domain.chat.UpdateMessageUseCase import com.example.appfunctions.agent.domain.chat.UpdateThreadParams import com.example.appfunctions.agent.domain.chat.UpdateThreadUseCase import com.example.appfunctions.agent.domain.pendingintent.SavePendingIntentUseCase +import javax.inject.Inject +import javax.inject.Singleton import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.flow.MutableStateFlow @@ -45,353 +48,378 @@ import kotlinx.coroutines.flow.filterNotNull import kotlinx.coroutines.flow.first import kotlinx.coroutines.flow.stateIn import kotlinx.coroutines.withContext -import javax.inject.Inject -import javax.inject.Singleton +import org.json.JSONObject /** Orchestrates the interaction between the LLM, AppFunctions, and the chat repository. */ @Singleton class AgentOrchestrator - @Inject - constructor( - private val manageThreadsUseCase: ManageThreadsUseCase, - private val observePendingMessagesUseCase: ObservePendingMessagesUseCase, - private val sendMessageUseCase: SendMessageUseCase, - private val updateMessageUseCase: UpdateMessageUseCase, - private val updateThreadUseCase: UpdateThreadUseCase, - private val llmProviderFactory: LlmProviderFactory, - private val settingsRepository: SettingsRepository, - private val getAppFunctionsUseCase: GetAppFunctionsUseCase, - private val convertInputToAppFunctionDataUseCase: ConvertInputToAppFunctionDataUseCase, - private val executeAppFunctionUseCase: ExecuteAppFunctionUseCase, - private val savePendingIntentUseCase: SavePendingIntentUseCase, - ) { - private val _status = MutableStateFlow(AgentStatus.Idle) - - /** The current status of the agent. */ - val status: StateFlow = _status.asStateFlow() - - /** - * Observes messages for a specific thread and processes any pending agent responses. This - * method suspends and collects messages until cancelled. - */ - suspend fun observeAndProcessMessages(threadId: String) = - coroutineScope { - val threadStateFlow = - manageThreadsUseCase.getThread(threadId).stateIn(this, SharingStarted.Eagerly, null) - - observePendingMessagesUseCase(threadId).collect { message -> - if (message != null) { - val thread = threadStateFlow.filterNotNull().first() - processMessage(message, thread) - } - } +@Inject +constructor( + private val manageThreadsUseCase: ManageThreadsUseCase, + private val observePendingMessagesUseCase: ObservePendingMessagesUseCase, + private val sendMessageUseCase: SendMessageUseCase, + private val updateMessageUseCase: UpdateMessageUseCase, + private val updateThreadUseCase: UpdateThreadUseCase, + private val llmProviderFactory: LlmProviderFactory, + private val settingsRepository: SettingsRepository, + private val getAppFunctionsUseCase: GetAppFunctionsUseCase, + private val convertInputToAppFunctionDataUseCase: ConvertInputToAppFunctionDataUseCase, + private val executeAppFunctionUseCase: ExecuteAppFunctionUseCase, + private val savePendingIntentUseCase: SavePendingIntentUseCase +) { + private val _status = MutableStateFlow(AgentStatus.Idle) + + /** The current status of the agent. */ + val status: StateFlow = _status.asStateFlow() + + /** + * Observes messages for a specific thread and processes any pending agent responses. This + * method suspends and collects messages until cancelled. + */ + suspend fun observeAndProcessMessages(threadId: String) = coroutineScope { + val threadStateFlow = + manageThreadsUseCase.getThread( + threadId + ).stateIn(this, SharingStarted.Eagerly, null) + + observePendingMessagesUseCase(threadId).collect { message -> + if (message != null) { + val thread = threadStateFlow.filterNotNull().first() + processMessage(message, thread) } + } + } - private suspend fun processMessage( - message: MessageEntity, - thread: ThreadEntity, - ) { - _status.value = AgentStatus.Thinking - - try { - val provider = thread.llmModel.providerName - val apiKey = getApiKey(provider) - if (apiKey == null) { - completeMessageWithError( - message.messageId, - message.threadId, - "API key is missing for ${provider.name}", - ) - _status.value = AgentStatus.Idle - return - } - - val disconnectedApps = settingsRepository.disconnectedApps.first() - val allTools = getAppFunctionsUseCase().first().values.flatten() - - val targetPackageName = message.targetPackageName - val queryText = message.textContent - - val tools = filterTools(allTools, disconnectedApps, targetPackageName) - - runInteractionLoop( - message = message, - thread = thread, - apiKey = apiKey, - tools = tools, - initialInput = queryText, - ) + private suspend fun processMessage(message: MessageEntity, thread: ThreadEntity) { + _status.value = AgentStatus.Thinking - _status.value = AgentStatus.Idle - } catch (e: Exception) { - Log.e("AgentOrchestrator", "Error processing message", e) + try { + val provider = thread.llmModel.providerName + val apiKey = getApiKey(provider) + if (apiKey == null) { completeMessageWithError( message.messageId, message.threadId, - e.message ?: "Unknown error occurred", + "API key is missing for ${provider.name}" ) _status.value = AgentStatus.Idle + return } + + val disconnectedApps = settingsRepository.disconnectedApps.first() + val allTools = getAppFunctionsUseCase().first().values.flatten() + + val targetPackageName = message.targetPackageName + val queryText = message.textContent + + val tools = filterTools(allTools, disconnectedApps, targetPackageName) + + runInteractionLoop( + message = message, + thread = thread, + apiKey = apiKey, + tools = tools, + initialInput = queryText + ) + + _status.value = AgentStatus.Idle + } catch (e: Exception) { + Log.e("AgentOrchestrator", "Error processing message", e) + completeMessageWithError( + message.messageId, + message.threadId, + e.message ?: "Unknown error occurred" + ) + _status.value = AgentStatus.Idle } + } - private fun filterTools( - allTools: List, - disconnectedApps: Set, - targetPackageName: String?, - ): List { - return allTools.filter { metadata -> - metadata.isEnabled && - metadata.packageName !in disconnectedApps && - (targetPackageName == null || metadata.packageName == targetPackageName) - } + private fun filterTools( + allTools: List, + disconnectedApps: Set, + targetPackageName: String? + ): List { + return allTools.filter { metadata -> + metadata.isEnabled && + metadata.packageName !in disconnectedApps && + (targetPackageName == null || metadata.packageName == targetPackageName) } + } - private suspend fun runInteractionLoop( - message: MessageEntity, - thread: ThreadEntity, - apiKey: String, - tools: List, - initialInput: String, - ) { - val provider = thread.llmModel.providerName - val modelName = thread.llmModel.modelName - val llmProvider = llmProviderFactory.getProvider(provider) - - var previousInteractionId = thread.latestInteractionId - var currentToolOutputs = emptyList() - var continueLoop = true - var currentInput = initialInput - - while (continueLoop) { - val llmInput = prepareLlmInput(currentToolOutputs, currentInput) - - currentToolOutputs = emptyList() - val response = - llmProvider.generateResponse( - previousInteractionId = previousInteractionId, - input = llmInput, - tools = tools, - apiKey = apiKey, - modelName = modelName, - ) + private suspend fun runInteractionLoop( + message: MessageEntity, + thread: ThreadEntity, + apiKey: String, + tools: List, + initialInput: String + ) { + val provider = thread.llmModel.providerName + val modelName = thread.llmModel.modelName + val llmProvider = llmProviderFactory.getProvider(provider) + + var previousInteractionId = thread.latestInteractionId + var currentToolOutputs = emptyList() + var continueLoop = true + var currentInput = initialInput + val capturedAttachments = mutableListOf() + + while (continueLoop) { + val llmInput = prepareLlmInput(currentToolOutputs, currentInput) + + currentToolOutputs = emptyList() + val response = + llmProvider.generateResponse( + previousInteractionId = previousInteractionId, + input = llmInput, + tools = tools, + apiKey = apiKey, + modelName = modelName + ) - when (val handleResult = handleLlmResponse(response, message, tools)) { - is HandleResult.Continue -> { - currentToolOutputs = handleResult.toolOutputs - previousInteractionId = handleResult.interactionId - } + when ( + val handleResult = + handleLlmResponse(response, message, tools, capturedAttachments) + ) { + is HandleResult.Continue -> { + currentToolOutputs = handleResult.toolOutputs + previousInteractionId = handleResult.interactionId + } - is HandleResult.Stop -> { - continueLoop = false - } + is HandleResult.Stop -> { + continueLoop = false } } } + } - private suspend fun getApiKey(provider: LlmProviderName): String? { - return when (provider) { - LlmProviderName.GEMINI -> settingsRepository.geminiApiKey.first() - } + private suspend fun getApiKey(provider: LlmProviderName): String? { + return when (provider) { + LlmProviderName.GEMINI -> settingsRepository.geminiApiKey.first() } + } - private fun prepareLlmInput( - currentToolOutputs: List, - currentInput: String, - ): LlmInput { - return if (currentToolOutputs.isNotEmpty()) { - LlmInput.ToolResponse(currentToolOutputs) - } else { - LlmInput.UserMessage(currentInput) - } + private fun prepareLlmInput( + currentToolOutputs: List, + currentInput: String + ): LlmInput { + return if (currentToolOutputs.isNotEmpty()) { + LlmInput.ToolResponse(currentToolOutputs) + } else { + LlmInput.UserMessage(currentInput) } + } - private sealed class HandleResult { - data class Continue(val toolOutputs: List, val interactionId: String) : - HandleResult() + private sealed class HandleResult { + data class Continue(val toolOutputs: List, val interactionId: String) : + HandleResult() - object Stop : HandleResult() - } + object Stop : HandleResult() + } - private sealed class ExecuteToolCallsResult { - data class Success(val toolOutputs: List) : ExecuteToolCallsResult() + private sealed class ExecuteToolCallsResult { + data class Success(val toolOutputs: List) : ExecuteToolCallsResult() - data class PendingIntentAction( - val pendingIntentId: String, - val pendingIntent: android.app.PendingIntent, - ) : ExecuteToolCallsResult() + data class PendingIntentAction( + val pendingIntentId: String, + val pendingIntent: android.app.PendingIntent + ) : ExecuteToolCallsResult() - object Error : ExecuteToolCallsResult() - } + object Error : ExecuteToolCallsResult() + } - private suspend fun handleLlmResponse( - response: LlmResponse, - message: MessageEntity, - tools: List, - ): HandleResult { - return when (response) { - is LlmResponse.Success -> { - updateThreadUseCase( - message.threadId, - UpdateThreadParams(interactionId = response.interactionId), - ) - updateMessageUseCase( - message.messageId, - UpdateMessageParams(status = MessageProcessingStatus.PROCESSED), - ) + private suspend fun handleLlmResponse( + response: LlmResponse, + message: MessageEntity, + tools: List, + capturedAttachments: MutableList + ): HandleResult { + return when (response) { + is LlmResponse.Success -> { + updateThreadUseCase( + message.threadId, + UpdateThreadParams(interactionId = response.interactionId) + ) + updateMessageUseCase( + message.messageId, + UpdateMessageParams(status = MessageProcessingStatus.PROCESSED) + ) - val toolCalls = response.parts.filterIsInstance() - val textParts = response.parts.filterIsInstance() - - val textContent = textParts.joinToString("\n") { it.text } - - if (toolCalls.isNotEmpty()) { - when (val toolResult = executeToolCalls(toolCalls, tools, message)) { - is ExecuteToolCallsResult.Success -> { - if (textContent.isNotEmpty()) { - sendMessageUseCase( - threadId = message.threadId, - role = MessageRole.ASSISTANT, - textContent = textContent, - processingStatus = MessageProcessingStatus.PROCESSED, - ) + val toolCalls = response.parts.filterIsInstance() + val textParts = response.parts.filterIsInstance() + + val textContent = textParts.joinToString("\n") { it.text } + + if (toolCalls.isNotEmpty()) { + when (val toolResult = executeToolCalls(toolCalls, tools, message)) { + is ExecuteToolCallsResult.Success -> { + for (output in toolResult.toolOutputs) { + runCatching { + val json = JSONObject(output.result) + val uri = json.optString("imageUri") + if (uri.isNotBlank()) { + val mimeType = + json.optString("mimeType") + .takeIf { it.isNotBlank() } + ?: "image/png" + capturedAttachments.add( + MessageAttachment(uri = uri, mimeType = mimeType) + ) + } } - HandleResult.Continue(toolResult.toolOutputs, response.interactionId) } - - is ExecuteToolCallsResult.PendingIntentAction -> { - savePendingIntentUseCase( - toolResult.pendingIntentId, - toolResult.pendingIntent, - ) + if (textContent.isNotEmpty()) { sendMessageUseCase( threadId = message.threadId, role = MessageRole.ASSISTANT, textContent = textContent, - processingStatus = MessageProcessingStatus.PROCESSED, - pendingIntentId = toolResult.pendingIntentId, + processingStatus = MessageProcessingStatus.PROCESSED ) - HandleResult.Stop - } - - is ExecuteToolCallsResult.Error -> { - HandleResult.Stop } + HandleResult.Continue( + toolResult.toolOutputs, + response.interactionId + ) } - } else { - if (textContent.isNotEmpty()) { + + is ExecuteToolCallsResult.PendingIntentAction -> { + savePendingIntentUseCase( + toolResult.pendingIntentId, + toolResult.pendingIntent + ) sendMessageUseCase( threadId = message.threadId, role = MessageRole.ASSISTANT, textContent = textContent, processingStatus = MessageProcessingStatus.PROCESSED, + pendingIntentId = toolResult.pendingIntentId ) + HandleResult.Stop } - HandleResult.Stop - } - } - is LlmResponse.Error -> { - Log.e("AgentOrchestrator", "LLM Error: ${response.errorMessage}") - completeMessageWithError(message.messageId, message.threadId, response.errorMessage) - _status.value = AgentStatus.Idle + is ExecuteToolCallsResult.Error -> { + HandleResult.Stop + } + } + } else { + if (textContent.isNotEmpty() || capturedAttachments.isNotEmpty()) { + sendMessageUseCase( + threadId = message.threadId, + role = MessageRole.ASSISTANT, + textContent = textContent, + processingStatus = MessageProcessingStatus.PROCESSED, + attachments = capturedAttachments + ) + } HandleResult.Stop } } - } - private suspend fun executeToolCalls( - toolCalls: List, - tools: List, - message: MessageEntity, - ): ExecuteToolCallsResult { - val results = mutableListOf() - for (toolCall in toolCalls) { - _status.value = AgentStatus.InvokingTool(toolCall.functionId, toolCall.packageName) - - val matchingTool = - tools.find { - it.packageName == toolCall.packageName && it.id == toolCall.functionId - } + is LlmResponse.Error -> { + Log.e("AgentOrchestrator", "LLM Error: ${response.errorMessage}") + completeMessageWithError( + message.messageId, + message.threadId, + response.errorMessage + ) + _status.value = AgentStatus.Idle + HandleResult.Stop + } + } + } - if (matchingTool == null) { - completeMessageWithError( - message.messageId, - message.threadId, - "Tool not found: ${toolCall.functionId}", - ) - return ExecuteToolCallsResult.Error + private suspend fun executeToolCalls( + toolCalls: List, + tools: List, + message: MessageEntity + ): ExecuteToolCallsResult { + val results = mutableListOf() + for (toolCall in toolCalls) { + _status.value = AgentStatus.InvokingTool(toolCall.functionId, toolCall.packageName) + + val matchingTool = + tools.find { + it.packageName == toolCall.packageName && it.id == toolCall.functionId } - val convertedInputs = toolCall.arguments.filterValues { it != null } as Map + if (matchingTool == null) { + completeMessageWithError( + message.messageId, + message.threadId, + "Tool not found: ${toolCall.functionId}" + ) + return ExecuteToolCallsResult.Error + } - val appFunctionDataResult = - withContext(Dispatchers.Default) { - convertInputToAppFunctionDataUseCase( - parameters = matchingTool.parameters, - components = matchingTool.components, - inputs = convertedInputs, - ) - } + val convertedInputs = toolCall.arguments.filterValues { it != null } as Map - if (appFunctionDataResult.isFailure) { - completeMessageWithError( - message.messageId, - message.threadId, - "Failed to convert arguments for ${toolCall.functionId}\n ${appFunctionDataResult.exceptionOrNull()}", + val appFunctionDataResult = + withContext(Dispatchers.Default) { + convertInputToAppFunctionDataUseCase( + parameters = matchingTool.parameters, + components = matchingTool.components, + inputs = convertedInputs ) - return ExecuteToolCallsResult.Error } - val executionResult = - executeAppFunctionUseCase( - function = matchingTool, - parameters = appFunctionDataResult.getOrThrow(), - threadId = message.threadId, - ) + if (appFunctionDataResult.isFailure) { + completeMessageWithError( + message.messageId, + message.threadId, + "Failed to convert arguments for ${toolCall.functionId}\n ${appFunctionDataResult.exceptionOrNull()}" + ) + return ExecuteToolCallsResult.Error + } - when (executionResult) { - is ExecuteAppFunctionResult.Data -> { - results.add( - ToolOutput( - functionId = toolCall.functionId, - callId = toolCall.callId, - result = executionResult.formattedJson, - ), - ) - } + val executionResult = + executeAppFunctionUseCase( + function = matchingTool, + parameters = appFunctionDataResult.getOrThrow(), + threadId = message.threadId + ) - is ExecuteAppFunctionResult.PendingIntentAction -> { - val pendingIntentId = java.util.UUID.randomUUID().toString() - return ExecuteToolCallsResult.PendingIntentAction( - pendingIntentId, - executionResult.pendingIntent, + when (executionResult) { + is ExecuteAppFunctionResult.Data -> { + results.add( + ToolOutput( + functionId = toolCall.functionId, + callId = toolCall.callId, + result = executionResult.formattedJson ) - } + ) + } - is ExecuteAppFunctionResult.Error -> - throw IllegalStateException( - "Tool execution failed for ${toolCall.functionId}: ${executionResult.exception.message}", - executionResult.exception, - ) + is ExecuteAppFunctionResult.PendingIntentAction -> { + val pendingIntentId = java.util.UUID.randomUUID().toString() + return ExecuteToolCallsResult.PendingIntentAction( + pendingIntentId, + executionResult.pendingIntent + ) } + + is ExecuteAppFunctionResult.Error -> + throw IllegalStateException( + "Tool execution failed for ${toolCall.functionId}: ${executionResult.exception.message}", + executionResult.exception + ) } - return ExecuteToolCallsResult.Success(results) } + return ExecuteToolCallsResult.Success(results) + } - private suspend fun completeMessageWithError( - messageId: String, - threadId: String, - reason: String, - ) { - updateMessageUseCase( - messageId, - UpdateMessageParams(status = MessageProcessingStatus.PROCESSED), - ) - sendMessageUseCase( - threadId = threadId, - role = MessageRole.ASSISTANT, - textContent = reason, - processingStatus = MessageProcessingStatus.FAILED, - ) - } + private suspend fun completeMessageWithError( + messageId: String, + threadId: String, + reason: String + ) { + updateMessageUseCase( + messageId, + UpdateMessageParams(status = MessageProcessingStatus.PROCESSED) + ) + sendMessageUseCase( + threadId = threadId, + role = MessageRole.ASSISTANT, + textContent = reason, + processingStatus = MessageProcessingStatus.FAILED + ) } +} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/chat/SendMessageUseCase.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/chat/SendMessageUseCase.kt index 91c3518..ef11560 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/chat/SendMessageUseCase.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/chat/SendMessageUseCase.kt @@ -16,6 +16,7 @@ package com.example.appfunctions.agent.domain.chat import com.example.appfunctions.agent.data.ChatRepository +import com.example.appfunctions.agent.data.db.entities.MessageAttachment import com.example.appfunctions.agent.data.db.entities.MessageEntity import com.example.appfunctions.agent.data.db.entities.MessageProcessingStatus import com.example.appfunctions.agent.data.db.entities.MessageRole @@ -24,37 +25,39 @@ import javax.inject.Inject /** Use case to send a message in a chat thread. */ class SendMessageUseCase - @Inject - constructor( - private val chatRepository: ChatRepository, +@Inject +constructor( + private val chatRepository: ChatRepository +) { + /** + * Executes the use case. + * + * @param threadId The ID of the thread to send the message to. + * @param role The role of the sender (USER or ASSISTANT). + * @param textContent The content of the message. + * @param processingStatus The processing status of the message. + */ + suspend operator fun invoke( + threadId: String, + role: MessageRole, + textContent: String, + processingStatus: MessageProcessingStatus, + pendingIntentId: String? = null, + targetPackageName: String? = null, + attachments: List = emptyList() ) { - /** - * Executes the use case. - * - * @param threadId The ID of the thread to send the message to. - * @param role The role of the sender (USER or ASSISTANT). - * @param textContent The content of the message. - * @param processingStatus The processing status of the message. - */ - suspend operator fun invoke( - threadId: String, - role: MessageRole, - textContent: String, - processingStatus: MessageProcessingStatus, - pendingIntentId: String? = null, - targetPackageName: String? = null, - ) { - val message = - MessageEntity( - messageId = UUID.randomUUID().toString(), - threadId = threadId, - role = role, - textContent = textContent, - timestamp = System.currentTimeMillis(), - processingStatus = processingStatus, - pendingIntentId = pendingIntentId, - targetPackageName = targetPackageName, - ) - chatRepository.sendMessage(message) - } + val message = + MessageEntity( + messageId = UUID.randomUUID().toString(), + threadId = threadId, + role = role, + textContent = textContent, + timestamp = System.currentTimeMillis(), + processingStatus = processingStatus, + pendingIntentId = pendingIntentId, + targetPackageName = targetPackageName, + attachments = attachments + ) + chatRepository.sendMessage(message) } +} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt index 327c8cf..6c9758a 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt @@ -88,6 +88,7 @@ import androidx.compose.ui.input.key.KeyEventType import androidx.compose.ui.input.key.key import androidx.compose.ui.input.key.onPreviewKeyEvent import androidx.compose.ui.input.key.type +import androidx.compose.ui.layout.ContentScale import androidx.compose.ui.platform.LocalConfiguration import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.platform.LocalDensity @@ -118,6 +119,7 @@ import androidx.compose.ui.window.PopupProperties import androidx.core.graphics.drawable.toBitmap import androidx.hilt.navigation.compose.hiltViewModel import androidx.lifecycle.compose.collectAsStateWithLifecycle +import coil.compose.AsyncImage import com.example.appfunctions.agent.R import com.example.appfunctions.agent.data.LlmModel import com.example.appfunctions.agent.data.db.entities.MessageEntity @@ -143,7 +145,7 @@ fun AgentDemoScreen(viewModel: AgentDemoViewModel = hiltViewModel()) { fun AgentDemoContent( uiState: AgentUiState, onEvent: (AgentUiEvent) -> Unit, - initialSidePanelVisible: Boolean = false, + initialSidePanelVisible: Boolean = false ) { val context = LocalContext.current val packageManager = context.packageManager @@ -172,7 +174,7 @@ fun AgentDemoContent( drawerState = drawerState, scope = scope, packageManager = packageManager, - initialSidePanelVisible = initialSidePanelVisible, + initialSidePanelVisible = initialSidePanelVisible ) } } @@ -188,7 +190,7 @@ fun AgentDemoContent( drawerState = drawerState, drawerContent = { ModalDrawerSheet( - drawerContainerColor = MaterialTheme.colorScheme.surface, + drawerContainerColor = MaterialTheme.colorScheme.surface ) { ChatHistorySidePanel( threads = threads, @@ -196,10 +198,10 @@ fun AgentDemoContent( onEvent = { event -> onEvent(event) scope.launch { drawerState.close() } - }, + } ) } - }, + } ) { content() } @@ -222,7 +224,7 @@ fun AgentDemoLoadedScreen( drawerState: DrawerState, scope: CoroutineScope, packageManager: PackageManager, - initialSidePanelVisible: Boolean = false, + initialSidePanelVisible: Boolean = false ) { var messageText by remember { mutableStateOf(TextFieldValue("")) } var isSidePanelVisible by remember { mutableStateOf(initialSidePanelVisible) } @@ -241,13 +243,13 @@ fun AgentDemoLoadedScreen( topBar = { Row( modifier = Modifier.padding(horizontal = 8.dp, vertical = 16.dp), - verticalAlignment = Alignment.CenterVertically, + verticalAlignment = Alignment.CenterVertically ) { ModelDropdown( modifier = - Modifier - .weight(1f) - .padding(horizontal = 8.dp), + Modifier + .weight(1f) + .padding(horizontal = 8.dp), currentThread = uiState.currentThread, onModelSelected = { onEvent(AgentUiEvent.OnModelSelected(it)) }, onMenuClick = { @@ -256,39 +258,39 @@ fun AgentDemoLoadedScreen( } else { scope.launch { drawerState.open() } } - }, + } ) IconButton( onClick = { onEvent(AgentUiEvent.OnCreateThread(uiState.currentThread.llmModel)) }, - modifier = Modifier.padding(horizontal = 8.dp), + modifier = Modifier.padding(horizontal = 8.dp) ) { Icon(imageVector = Icons.Default.Add, contentDescription = "Create Thread") } } - }, + } ) { paddingValues -> Row( modifier = - Modifier - .fillMaxSize() - .imePadding() - .padding( - top = paddingValues.calculateTopPadding(), - ), + Modifier + .fillMaxSize() + .imePadding() + .padding( + top = paddingValues.calculateTopPadding() + ) ) { // Side Panel (only for wide screens) if (isWideScreen) { AnimatedVisibility( visible = isSidePanelVisible, enter = slideInHorizontally() + expandHorizontally(), - exit = slideOutHorizontally() + shrinkHorizontally(), + exit = slideOutHorizontally() + shrinkHorizontally() ) { ChatHistorySidePanel( threads = uiState.threads, currentThread = uiState.currentThread, - onEvent = onEvent, + onEvent = onEvent ) } } @@ -296,20 +298,20 @@ fun AgentDemoLoadedScreen( // Main Chat Area Column( modifier = - Modifier - .weight(1f) - .fillMaxHeight() - .padding(start = 16.dp, end = 16.dp), - verticalArrangement = Arrangement.SpaceBetween, + Modifier + .weight(1f) + .fillMaxHeight() + .padding(start = 16.dp, end = 16.dp), + verticalArrangement = Arrangement.SpaceBetween ) { // Messages List LazyColumn( modifier = - Modifier - .weight(1f) - .fillMaxWidth() - .clip(RoundedCornerShape(16.dp)), - reverseLayout = true, + Modifier + .weight(1f) + .fillMaxWidth() + .clip(RoundedCornerShape(16.dp)), + reverseLayout = true ) { // Status item at the bottom (above input) if not // idle @@ -317,21 +319,21 @@ fun AgentDemoLoadedScreen( item { StatusIndicator( status = uiState.status, - packageManager = packageManager, + packageManager = packageManager ) } } items( items = uiState.messages.reversed(), - key = { message -> message.messageId }, + key = { message -> message.messageId } ) { message -> MessageBubble( message = message, isValidAction = - message.pendingIntentId in uiState.activePendingActionIds, + message.pendingIntentId in uiState.activePendingActionIds, installedApps = uiState.installedApps, - onConfirmAction = { onEvent(AgentUiEvent.OnConfirmAction(it)) }, + onConfirmAction = { onEvent(AgentUiEvent.OnConfirmAction(it)) } ) } } @@ -376,12 +378,12 @@ fun AgentDemoLoadedScreen( anchorBounds: IntRect, windowSize: IntSize, layoutDirection: LayoutDirection, - popupContentSize: IntSize, + popupContentSize: IntSize ): IntOffset { val gap = with(density) { 2.dp.roundToPx() } return IntOffset( x = anchorBounds.left, - y = anchorBounds.top - popupContentSize.height - gap, + y = anchorBounds.top - popupContentSize.height - gap ) } } @@ -412,61 +414,61 @@ fun AgentDemoLoadedScreen( } }, modifier = - Modifier - .fillMaxWidth() - .padding(vertical = 16.dp) - .onPreviewKeyEvent { keyEvent -> - if ( - (keyEvent.key == Key.Enter || keyEvent.key == Key.NumPadEnter) && - keyEvent.type == KeyEventType.KeyDown - ) { - sendMessage() - true - } else { - false - } - }, + Modifier + .fillMaxWidth() + .padding(vertical = 16.dp) + .onPreviewKeyEvent { keyEvent -> + if ( + (keyEvent.key == Key.Enter || keyEvent.key == Key.NumPadEnter) && + keyEvent.type == KeyEventType.KeyDown + ) { + sendMessage() + true + } else { + false + } + }, enabled = uiState.status == AgentStatus.Idle, shape = CircleShape, placeholder = { Text(stringResource(R.string.agent_demo_ask_agent)) }, visualTransformation = visualTransformation, colors = - OutlinedTextFieldDefaults.colors( - unfocusedContainerColor = MaterialTheme.colorScheme.surfaceBright, - focusedContainerColor = MaterialTheme.colorScheme.surfaceBright, - unfocusedBorderColor = Color.Transparent, - focusedBorderColor = Color.Transparent, - ), + OutlinedTextFieldDefaults.colors( + unfocusedContainerColor = MaterialTheme.colorScheme.surfaceBright, + focusedContainerColor = MaterialTheme.colorScheme.surfaceBright, + unfocusedBorderColor = Color.Transparent, + focusedBorderColor = Color.Transparent + ), trailingIcon = { IconButton( onClick = sendMessage, enabled = - messageText.text.isNotBlank() && - uiState.status == AgentStatus.Idle, + messageText.text.isNotBlank() && + uiState.status == AgentStatus.Idle ) { Icon( imageVector = Icons.AutoMirrored.Filled.Send, contentDescription = - stringResource(R.string.agent_demo_send), + stringResource(R.string.agent_demo_send) ) } - }, + } ) if (showAutocomplete && filteredApps.isNotEmpty()) { Popup( popupPositionProvider = popupPositionProvider, onDismissRequest = {}, - properties = PopupProperties(focusable = false), + properties = PopupProperties(focusable = false) ) { Card( modifier = Modifier.fillMaxWidth(0.9f), elevation = CardDefaults.cardElevation(defaultElevation = 8.dp), colors = - CardDefaults.cardColors( - containerColor = MaterialTheme.colorScheme.surfaceBright, - ), - shape = MaterialTheme.shapes.medium, + CardDefaults.cardColors( + containerColor = MaterialTheme.colorScheme.surfaceBright + ), + shape = MaterialTheme.shapes.medium ) { Column(modifier = Modifier.padding(vertical = 4.dp)) { filteredApps.take(5).forEach { app -> @@ -477,18 +479,18 @@ fun AgentDemoLoadedScreen( val selectionStart = messageText.selection.start val textBeforeCursor = currentText.take( - selectionStart, + selectionStart ) val textAfterCursor = currentText.drop( - selectionStart, + selectionStart ) val mentionIndex = textBeforeCursor.lastIndexOf('@') if (mentionIndex >= 0) { val textBeforeMention = textBeforeCursor.substring( 0, - mentionIndex, + mentionIndex ) val newText = "$textBeforeMention@${app.label} $textAfterCursor" @@ -498,13 +500,13 @@ fun AgentDemoLoadedScreen( TextFieldValue( text = newText, selection = - TextRange( - newCursorPosition, - ), + TextRange( + newCursorPosition + ) ) selectedAppPackageName = app.packageName } - }, + } ) } } @@ -523,19 +525,19 @@ fun ModelDropdown( modifier: Modifier = Modifier, currentThread: ThreadEntity?, onModelSelected: (LlmModel) -> Unit, - onMenuClick: () -> Unit, + onMenuClick: () -> Unit ) { var expanded by remember { mutableStateOf(false) } ExposedDropdownMenuBox( modifier = modifier, expanded = expanded, - onExpandedChange = { expanded = !expanded }, + onExpandedChange = { expanded = !expanded } ) { Surface( modifier = Modifier.padding(bottom = 8.dp), shadowElevation = 2.dp, shape = CircleShape, - color = MaterialTheme.colorScheme.surfaceBright, + color = MaterialTheme.colorScheme.surfaceBright ) { val text = currentThread?.llmModel?.modelName @@ -549,38 +551,38 @@ fun ModelDropdown( Row( modifier = - Modifier - .fillMaxWidth() - .height(56.dp) - .padding(start = 4.dp, end = 16.dp), - verticalAlignment = Alignment.CenterVertically, + Modifier + .fillMaxWidth() + .height(56.dp) + .padding(start = 4.dp, end = 16.dp), + verticalAlignment = Alignment.CenterVertically ) { IconButton(onClick = onMenuClick) { Icon(imageVector = Icons.Default.Menu, contentDescription = "Menu") } Row( modifier = - Modifier - .weight(1f) - .fillMaxHeight() - .menuAnchor( - ExposedDropdownMenuAnchorType.PrimaryEditable, - enabled = true, - ), - verticalAlignment = Alignment.CenterVertically, + Modifier + .weight(1f) + .fillMaxHeight() + .menuAnchor( + ExposedDropdownMenuAnchorType.PrimaryEditable, + enabled = true + ), + verticalAlignment = Alignment.CenterVertically ) { Column(modifier = Modifier.weight(1f)) { Text( text = stringResource(R.string.agent_demo_title), style = MaterialTheme.typography.labelMedium, - color = MaterialTheme.colorScheme.onSurfaceVariant, + color = MaterialTheme.colorScheme.onSurfaceVariant ) Text( text = text, style = MaterialTheme.typography.bodyMedium, color = textColor, maxLines = 1, - overflow = TextOverflow.Ellipsis, + overflow = TextOverflow.Ellipsis ) } Icon(imageVector = Icons.Default.ArrowDropDown, contentDescription = null) @@ -593,21 +595,21 @@ fun ModelDropdown( onDismissRequest = { expanded = false }, modifier = Modifier.exposedDropdownSize(), containerColor = MaterialTheme.colorScheme.surfaceBright, - shape = RoundedCornerShape(28.dp), + shape = RoundedCornerShape(28.dp) ) { item { Text( "--- Gemini ---", color = MaterialTheme.colorScheme.secondary, modifier = Modifier.padding(horizontal = 24.dp, vertical = 8.dp), - style = MaterialTheme.typography.labelLarge, + style = MaterialTheme.typography.labelLarge ) } val models = listOf( LlmModel.GEMINI_3_1_PRO_PREVIEW, LlmModel.GEMINI_3_FLASH_PREVIEW, - LlmModel.GEMINI_3_1_FLASH_LITE_PREVIEW, + LlmModel.GEMINI_3_1_FLASH_LITE_PREVIEW ) items(models) { model -> DropdownMenuItem( @@ -616,7 +618,7 @@ fun ModelDropdown( onModelSelected(model) expanded = false }, - contentPadding = PaddingValues(horizontal = 24.dp, vertical = 8.dp), + contentPadding = PaddingValues(horizontal = 24.dp, vertical = 8.dp) ) } } @@ -628,7 +630,7 @@ fun MessageBubble( message: MessageEntity, isValidAction: Boolean, installedApps: List, - onConfirmAction: (String) -> Unit, + onConfirmAction: (String) -> Unit ) { val alignment = if (message.role == MessageRole.USER) Alignment.End else Alignment.Start val isError = message.processingStatus == MessageProcessingStatus.FAILED @@ -647,15 +649,15 @@ fun MessageBubble( Column( modifier = - Modifier - .fillMaxWidth() - .padding(vertical = 4.dp, horizontal = 2.dp), - horizontalAlignment = alignment, + Modifier + .fillMaxWidth() + .padding(vertical = 4.dp, horizontal = 2.dp), + horizontalAlignment = alignment ) { Surface( shape = MaterialTheme.shapes.large, color = backgroundColor, - shadowElevation = if (message.role == MessageRole.ASSISTANT) 1.dp else 0.dp, + shadowElevation = if (message.role == MessageRole.ASSISTANT) 1.dp else 0.dp ) { Column(modifier = Modifier.padding(12.dp)) { SelectionContainer { @@ -664,20 +666,24 @@ fun MessageBubble( Icon( imageVector = Icons.Filled.Warning, contentDescription = stringResource(R.string.debugging_error), - tint = textColor, + tint = textColor ) Spacer(modifier = Modifier.width(8.dp)) } val contentText = - if (message.textContent.isEmpty() && + if ( + message.textContent.isEmpty() && message.pendingIntentId != null ) { stringResource(R.string.agent_demo_action_confirmation_needed) } else { message.textContent } + if (message.role != MessageRole.USER) { - Markdown(content = contentText) + if (contentText.isNotEmpty()) { + Markdown(content = contentText) + } } else { val chipBgColor = MaterialTheme.colorScheme.primary val chipTextColor = MaterialTheme.colorScheme.onPrimary @@ -695,16 +701,19 @@ fun MessageBubble( installedApps, chipBgColor, chipTextColor, - density, + density ) { val map = mutableMapOf() if (installedApps.isNotEmpty() && contentText.contains("@")) { val appLabelsPattern = installedApps.joinToString( - "|", + "|" ) { Regex.escape(it.label) } val regex = - Regex("@($appLabelsPattern)\\b", RegexOption.IGNORE_CASE) + Regex( + "@($appLabelsPattern)\\b", + RegexOption.IGNORE_CASE + ) regex.findAll(contentText).forEachIndexed { index, match -> val id = "chip_$index" val appName = match.value @@ -712,17 +721,17 @@ fun MessageBubble( textMeasurer.measure( text = appName, style = - typographyStyle.copy( - fontWeight = FontWeight.Bold, - ), + typographyStyle.copy( + fontWeight = FontWeight.Bold + ) ) val widthSp = with( - density, + density ) { (measured.size.width + 8.dp.roundToPx()).toSp() } val heightSp = with( - density, + density ) { (measured.size.height + 2.dp.roundToPx()).toSp() } map[id] = @@ -730,29 +739,29 @@ fun MessageBubble( Placeholder( width = widthSp, height = heightSp, - placeholderVerticalAlign = PlaceholderVerticalAlign.TextCenter, - ), + placeholderVerticalAlign = PlaceholderVerticalAlign.TextCenter + ) ) { Surface( shape = - androidx.compose.foundation.shape.RoundedCornerShape( - 6.dp, - ), - color = chipBgColor, + androidx.compose.foundation.shape.RoundedCornerShape( + 6.dp + ), + color = chipBgColor ) { Box(contentAlignment = Alignment.Center) { Text( text = appName, color = chipTextColor, style = - typographyStyle.copy( - fontWeight = FontWeight.Bold, - ), + typographyStyle.copy( + fontWeight = FontWeight.Bold + ), modifier = - Modifier.padding( - horizontal = 4.dp, - vertical = 1.dp, - ), + Modifier.padding( + horizontal = 4.dp, + vertical = 1.dp + ) ) } } @@ -766,7 +775,7 @@ fun MessageBubble( text = formattedText, inlineContent = inlineContentMap, color = textColor, - style = typographyStyle, + style = typographyStyle ) } } @@ -779,44 +788,59 @@ fun MessageBubble( enabled = isValidAction, shape = CircleShape, colors = - ButtonDefaults.buttonColors( - containerColor = MaterialTheme.colorScheme.primary, - contentColor = MaterialTheme.colorScheme.onPrimary, - ), + ButtonDefaults.buttonColors( + containerColor = MaterialTheme.colorScheme.primary, + contentColor = MaterialTheme.colorScheme.onPrimary + ) ) { Text( if (isValidAction) { stringResource(R.string.agent_demo_confirm_action) } else { stringResource(R.string.agent_demo_action_expired) - }, + } ) } } } } + + if (message.role != MessageRole.USER) { + message.attachments.forEach { attachment -> + if (attachment.mimeType.startsWith("image/", ignoreCase = true)) { + Spacer(modifier = Modifier.height(6.dp)) + AsyncImage( + model = attachment.uri, + contentDescription = "Generated Image", + modifier = + Modifier + .fillMaxWidth(0.85f) + .height(240.dp) + .clip(RoundedCornerShape(16.dp)), + contentScale = ContentScale.Crop + ) + } + } + } } } @Composable -fun StatusIndicator( - status: AgentStatus, - packageManager: PackageManager, -) { +fun StatusIndicator(status: AgentStatus, packageManager: PackageManager) { when (status) { AgentStatus.Thinking -> { Row( modifier = - Modifier - .fillMaxWidth() - .padding(vertical = 8.dp), - verticalAlignment = Alignment.CenterVertically, + Modifier + .fillMaxWidth() + .padding(vertical = 8.dp), + verticalAlignment = Alignment.CenterVertically ) { CircularProgressIndicator(modifier = Modifier.size(24.dp)) Spacer(modifier = Modifier.width(8.dp)) Text( stringResource(R.string.agent_demo_thinking), - style = MaterialTheme.typography.bodyMedium, + style = MaterialTheme.typography.bodyMedium ) } } @@ -838,22 +862,22 @@ fun StatusIndicator( Surface( modifier = - Modifier - .fillMaxWidth() - .padding(vertical = 8.dp), + Modifier + .fillMaxWidth() + .padding(vertical = 8.dp), shape = MaterialTheme.shapes.large, color = MaterialTheme.colorScheme.surfaceBright, - shadowElevation = 2.dp, + shadowElevation = 2.dp ) { Row( modifier = Modifier.padding(12.dp), - verticalAlignment = Alignment.CenterVertically, + verticalAlignment = Alignment.CenterVertically ) { appIcon?.let { Image( bitmap = it.toBitmap().asImageBitmap(), contentDescription = null, - modifier = Modifier.size(40.dp), + modifier = Modifier.size(40.dp) ) Spacer(modifier = Modifier.width(12.dp)) } @@ -864,7 +888,7 @@ fun StatusIndicator( Spacer(modifier = Modifier.width(8.dp)) Text( stringResource(R.string.agent_demo_connecting), - style = MaterialTheme.typography.bodyMedium, + style = MaterialTheme.typography.bodyMedium ) } } @@ -883,26 +907,26 @@ fun ChatHistorySidePanel( threads: List, currentThread: ThreadEntity?, onEvent: (AgentUiEvent) -> Unit, - modifier: Modifier = Modifier, + modifier: Modifier = Modifier ) { Column( modifier = - modifier - .width(280.dp) - .fillMaxHeight() - .padding(16.dp), + modifier + .width(280.dp) + .fillMaxHeight() + .padding(16.dp) ) { Text( text = stringResource(R.string.agent_demo_chat_history), style = MaterialTheme.typography.titleLarge, color = MaterialTheme.colorScheme.onSurface, - modifier = Modifier.padding(bottom = 16.dp), + modifier = Modifier.padding(bottom = 16.dp) ) LazyColumn(modifier = Modifier.fillMaxSize()) { items( items = threads, - key = { thread -> thread.threadId }, + key = { thread -> thread.threadId } ) { thread -> val isSelected = thread.threadId == currentThread?.threadId val backgroundColor = @@ -920,26 +944,26 @@ fun ChatHistorySidePanel( Surface( modifier = - Modifier - .fillMaxWidth() - .padding(vertical = 4.dp) - .clickable { - onEvent(AgentUiEvent.OnThreadSelected(thread.threadId)) - }, + Modifier + .fillMaxWidth() + .padding(vertical = 4.dp) + .clickable { + onEvent(AgentUiEvent.OnThreadSelected(thread.threadId)) + }, shape = MaterialTheme.shapes.medium, color = backgroundColor, - contentColor = textColor, + contentColor = textColor ) { Column(modifier = Modifier.padding(12.dp)) { Text( text = thread.llmModel.modelName, style = MaterialTheme.typography.bodyMedium, - color = textColor, + color = textColor ) Text( text = "ID: ${thread.threadId.take(8)}", style = MaterialTheme.typography.bodySmall, - color = textColor.copy(alpha = 0.7f), + color = textColor.copy(alpha = 0.7f) ) } } @@ -950,7 +974,7 @@ fun ChatHistorySidePanel( class InlineAppScopingVisualTransformation( private val installedApps: List, - private val chipTextColor: Color, + private val chipTextColor: Color ) : VisualTransformation { private val regex: Regex? = if (installedApps.isNotEmpty()) { @@ -978,8 +1002,8 @@ class InlineAppScopingVisualTransformation( withStyle( SpanStyle( color = chipTextColor, - fontWeight = FontWeight.Bold, - ), + fontWeight = FontWeight.Bold + ) ) { append(match.value) } @@ -994,10 +1018,7 @@ class InlineAppScopingVisualTransformation( } } -fun formatMessageText( - text: String, - installedApps: List, -): AnnotatedString { +fun formatMessageText(text: String, installedApps: List): AnnotatedString { if (installedApps.isEmpty() || !text.contains("@")) { return AnnotatedString(text) } diff --git a/agent/app/src/main/res/xml/file_paths.xml b/agent/app/src/main/res/xml/file_paths.xml new file mode 100644 index 0000000..5074b23 --- /dev/null +++ b/agent/app/src/main/res/xml/file_paths.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/data/db/entities/MessageAttachmentConverterTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/data/db/entities/MessageAttachmentConverterTest.kt new file mode 100644 index 0000000..5c665cb --- /dev/null +++ b/agent/app/src/test/java/com/example/appfunctions/agent/data/db/entities/MessageAttachmentConverterTest.kt @@ -0,0 +1,56 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.appfunctions.agent.data.db.entities + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test + +class MessageAttachmentConverterTest { + + private val converter = MessageAttachmentConverter() + + @Test + fun testSerializationAndDeserialization() { + val attachments = + listOf( + MessageAttachment( + uri = "content://com.example.appfunctions.agent.fileprovider/cache/test.jpg", + mimeType = "image/jpeg" + ), + MessageAttachment( + uri = "content://com.example.appfunctions.agent.fileprovider/cache/test.png", + mimeType = "image/png" + ) + ) + + val json = converter.fromAttachments(attachments) + val decoded = converter.toAttachments(json) + + assertEquals(attachments, decoded) + } + + @Test + fun testEmptyOrNullStringDeserializesToEmptyList() { + assertTrue(converter.toAttachments(null).isEmpty()) + assertTrue(converter.toAttachments("").isEmpty()) + } + + @Test + fun testMalformedJsonDeserializesToEmptyList() { + assertTrue(converter.toAttachments("{invalid_json").isEmpty()) + } +} diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt index 592d28d..5fbe62e 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt @@ -15,6 +15,7 @@ */ package com.example.appfunctions.agent.domain +import androidx.appfunctions.AppFunctionData import androidx.appfunctions.metadata.AppFunctionMetadata import androidx.appfunctions.metadata.AppFunctionPackageMetadata import com.example.appfunctions.agent.data.LlmModel @@ -25,6 +26,7 @@ import com.example.appfunctions.agent.data.db.entities.MessageProcessingStatus import com.example.appfunctions.agent.data.db.entities.MessageRole import com.example.appfunctions.agent.data.db.entities.ThreadEntity import com.example.appfunctions.agent.domain.appfunction.ConvertInputToAppFunctionDataUseCase +import com.example.appfunctions.agent.domain.appfunction.ExecuteAppFunctionResult import com.example.appfunctions.agent.domain.appfunction.ExecuteAppFunctionUseCase import com.example.appfunctions.agent.domain.appfunction.GetAppFunctionsUseCase import com.example.appfunctions.agent.domain.chat.ManageThreadsUseCase @@ -48,8 +50,10 @@ import org.junit.Before import org.junit.Test import org.junit.runner.RunWith import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config @RunWith(RobolectricTestRunner::class) +@Config(sdk = [34]) class AgentOrchestratorTest { private val observePendingMessagesUseCase: ObservePendingMessagesUseCase = mockk() private val updateMessageUseCase: UpdateMessageUseCase = mockk(relaxed = true) @@ -79,148 +83,145 @@ class AgentOrchestratorTest { getAppFunctionsUseCase = getAppFunctionsUseCase, convertInputToAppFunctionDataUseCase = convertInputToAppFunctionDataUseCase, executeAppFunctionUseCase = executeAppFunctionUseCase, - savePendingIntentUseCase = savePendingIntentUseCase, + savePendingIntentUseCase = savePendingIntentUseCase ) } @Test - fun `observeAndProcessMessages fails when API key is missing`() = - runTest { - val threadId = "thread_1" - val message = createUserMessage(threadId, "Hello", messageId = "msg_1") - val thread = createThread(threadId) + fun `observeAndProcessMessages fails when API key is missing`() = runTest { + val threadId = "thread_1" + val message = createUserMessage(threadId, "Hello", messageId = "msg_1") + val thread = createThread(threadId) - setupDefaultMocks(threadId, message, thread, apiKey = null) + setupDefaultMocks(threadId, message, thread, apiKey = null) - agentOrchestrator.observeAndProcessMessages(threadId) + agentOrchestrator.observeAndProcessMessages(threadId) - // Verify behavior (state) - assertEquals(AgentStatus.Idle, agentOrchestrator.status.value) + // Verify behavior (state) + assertEquals(AgentStatus.Idle, agentOrchestrator.status.value) - // Verify interactions - coVerify { - updateMessageUseCase( - message.messageId, - UpdateMessageParams(status = MessageProcessingStatus.PROCESSED), - ) - sendMessageUseCase( - threadId = threadId, - role = MessageRole.ASSISTANT, - textContent = "API key is missing for GEMINI", - processingStatus = MessageProcessingStatus.FAILED, - ) - } + // Verify interactions + coVerify { + updateMessageUseCase( + message.messageId, + UpdateMessageParams(status = MessageProcessingStatus.PROCESSED) + ) + sendMessageUseCase( + threadId = threadId, + role = MessageRole.ASSISTANT, + textContent = "API key is missing for GEMINI", + processingStatus = MessageProcessingStatus.FAILED + ) } + } @Test - fun `observeAndProcessMessages fails when LLM returns error`() = - runTest { - val threadId = "thread_1" - val message = createUserMessage(threadId, "Hello", messageId = "msg_1") - val thread = createThread(threadId) - val llmProvider = mockk() + fun `observeAndProcessMessages fails when LLM returns error`() = runTest { + val threadId = "thread_1" + val message = createUserMessage(threadId, "Hello", messageId = "msg_1") + val thread = createThread(threadId) + val llmProvider = mockk() - setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) - coEvery { getAppFunctionsUseCase() } returns flowOf(emptyMap()) + setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) + coEvery { getAppFunctionsUseCase() } returns flowOf(emptyMap()) - val errorMsg = "LLM failed" - coEvery { llmProvider.generateResponse(any(), any(), any(), any(), any()) } returns - LlmResponse.Error(errorMsg) + val errorMsg = "LLM failed" + coEvery { llmProvider.generateResponse(any(), any(), any(), any(), any()) } returns + LlmResponse.Error(errorMsg) - agentOrchestrator.observeAndProcessMessages(threadId) + agentOrchestrator.observeAndProcessMessages(threadId) - // Verify behavior (state) - assertEquals(AgentStatus.Idle, agentOrchestrator.status.value) + // Verify behavior (state) + assertEquals(AgentStatus.Idle, agentOrchestrator.status.value) - // Verify interactions - coVerify { - updateMessageUseCase( - message.messageId, - UpdateMessageParams(status = MessageProcessingStatus.PROCESSED), - ) - sendMessageUseCase( - threadId = threadId, - role = MessageRole.ASSISTANT, - textContent = errorMsg, - processingStatus = MessageProcessingStatus.FAILED, - ) - } + // Verify interactions + coVerify { + updateMessageUseCase( + message.messageId, + UpdateMessageParams(status = MessageProcessingStatus.PROCESSED) + ) + sendMessageUseCase( + threadId = threadId, + role = MessageRole.ASSISTANT, + textContent = errorMsg, + processingStatus = MessageProcessingStatus.FAILED + ) } + } @Test - fun `observeAndProcessMessages succeeds when LLM returns text`() = - runTest { - val threadId = "thread_1" - val message = createUserMessage(threadId, "Hello", messageId = "msg_1") - val thread = createThread(threadId) - val llmProvider = mockk() - - setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) - coEvery { getAppFunctionsUseCase() } returns flowOf(emptyMap()) - - val responseText = "Hi there" - coEvery { llmProvider.generateResponse(any(), any(), any(), any(), any()) } returns - LlmResponse.Success( - "interaction_123", - listOf(LlmResponsePart.Text(responseText)), - ) + fun `observeAndProcessMessages succeeds when LLM returns text`() = runTest { + val threadId = "thread_1" + val message = createUserMessage(threadId, "Hello", messageId = "msg_1") + val thread = createThread(threadId) + val llmProvider = mockk() + + setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) + coEvery { getAppFunctionsUseCase() } returns flowOf(emptyMap()) + + val responseText = "Hi there" + coEvery { llmProvider.generateResponse(any(), any(), any(), any(), any()) } returns + LlmResponse.Success( + "interaction_123", + listOf(LlmResponsePart.Text(responseText)) + ) - agentOrchestrator.observeAndProcessMessages(threadId) + agentOrchestrator.observeAndProcessMessages(threadId) - // Verify behavior (state) - assertEquals(AgentStatus.Idle, agentOrchestrator.status.value) + // Verify behavior (state) + assertEquals(AgentStatus.Idle, agentOrchestrator.status.value) - // Verify interactions - coVerify { - updateThreadUseCase(threadId, UpdateThreadParams(interactionId = "interaction_123")) - sendMessageUseCase( - threadId = threadId, - role = MessageRole.ASSISTANT, - textContent = responseText, - processingStatus = MessageProcessingStatus.PROCESSED, - ) - updateMessageUseCase( - message.messageId, - UpdateMessageParams(status = MessageProcessingStatus.PROCESSED), - ) - } + // Verify interactions + coVerify { + updateThreadUseCase(threadId, UpdateThreadParams(interactionId = "interaction_123")) + sendMessageUseCase( + threadId = threadId, + role = MessageRole.ASSISTANT, + textContent = responseText, + processingStatus = MessageProcessingStatus.PROCESSED + ) + updateMessageUseCase( + message.messageId, + UpdateMessageParams(status = MessageProcessingStatus.PROCESSED) + ) } + } @Test - fun `observeAndProcessMessages scopes tools when targetPackageName is set`() = - runTest { - val threadId = "thread_1" - val message = - createUserMessage( - threadId = threadId, - textContent = "run geo code address for n1c4ag", - targetPackageName = "com.google.android.appfunctiontestingagent", - ) - val thread = createThread(threadId) - val llmProvider = mockk() + fun `observeAndProcessMessages scopes tools when targetPackageName is set`() = runTest { + val threadId = "thread_1" + val message = + createUserMessage( + threadId = threadId, + textContent = "run geo code address for n1c4ag", + targetPackageName = "com.google.android.appfunctiontestingagent" + ) + val thread = createThread(threadId) + val llmProvider = mockk() - val tool1 = createMockTool("com.google.android.appfunctiontestingagent", "run_geo_code") - val tool2 = createMockTool("com.google.android.digitalwellbeing", "digital_well_being_tool") - mockAppFunctions(listOf(tool1, tool2)) + val tool1 = createMockTool("com.google.android.appfunctiontestingagent", "run_geo_code") + val tool2 = + createMockTool("com.google.android.digitalwellbeing", "digital_well_being_tool") + mockAppFunctions(listOf(tool1, tool2)) - setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) + setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) - coEvery { - llmProvider.generateResponse(any(), any(), any(), any(), any()) - } returns LlmResponse.Success("interaction_id", listOf(LlmResponsePart.Text("Success"))) + coEvery { + llmProvider.generateResponse(any(), any(), any(), any(), any()) + } returns LlmResponse.Success("interaction_id", listOf(LlmResponsePart.Text("Success"))) - agentOrchestrator.observeAndProcessMessages(threadId) + agentOrchestrator.observeAndProcessMessages(threadId) - coVerify { - llmProvider.generateResponse( - previousInteractionId = null, - input = eq(LlmInput.UserMessage("run geo code address for n1c4ag")), - tools = listOf(tool1), - apiKey = "dummy_key", - modelName = any(), - ) - } + coVerify { + llmProvider.generateResponse( + previousInteractionId = null, + input = eq(LlmInput.UserMessage("run geo code address for n1c4ag")), + tools = listOf(tool1), + apiKey = "dummy_key", + modelName = any() + ) } + } @Test fun `observeAndProcessMessages does not scope tools when targetPackageName is null`() = @@ -231,7 +232,8 @@ class AgentOrchestratorTest { val llmProvider = mockk() val tool1 = createMockTool("com.google.android.appfunctiontestingagent", "run_geo_code") - val tool2 = createMockTool("com.google.android.digitalwellbeing", "digital_well_being_tool") + val tool2 = + createMockTool("com.google.android.digitalwellbeing", "digital_well_being_tool") mockAppFunctions(listOf(tool1, tool2)) setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) @@ -248,7 +250,7 @@ class AgentOrchestratorTest { input = eq(LlmInput.UserMessage("run geo code address for n1c4ag")), tools = listOf(tool1, tool2), apiKey = "dummy_key", - modelName = any(), + modelName = any() ) } } @@ -257,7 +259,7 @@ class AgentOrchestratorTest { threadId: String, textContent: String, messageId: String = "message_1", - targetPackageName: String? = null, + targetPackageName: String? = null ) = MessageEntity( messageId = messageId, threadId = threadId, @@ -265,26 +267,26 @@ class AgentOrchestratorTest { textContent = textContent, timestamp = System.currentTimeMillis(), processingStatus = MessageProcessingStatus.PENDING_AGENT_RESPONSE, - targetPackageName = targetPackageName, + targetPackageName = targetPackageName ) private fun createThread( threadId: String, llmModel: LlmModel = LlmModel.GEMINI_3_FLASH_PREVIEW, - latestInteractionId: String? = null, + latestInteractionId: String? = null ) = ThreadEntity( threadId = threadId, createdAt = System.currentTimeMillis(), llmModel = llmModel, - latestInteractionId = latestInteractionId, + latestInteractionId = latestInteractionId ) private fun createMockTool( packageName: String, id: String, - isEnabled: Boolean = true, + isEnabled: Boolean = true ): AppFunctionMetadata { - val tool = mockk() + val tool = mockk(relaxed = true) every { tool.packageName } returns packageName every { tool.id } returns id every { tool.isEnabled } returns isEnabled @@ -302,7 +304,7 @@ class AgentOrchestratorTest { thread: ThreadEntity, apiKey: String? = "dummy_key", disconnectedApps: Set = emptySet(), - llmProvider: LlmProvider = mockk(), + llmProvider: LlmProvider = mockk() ) { coEvery { observePendingMessagesUseCase(threadId) } returns flow { @@ -314,4 +316,88 @@ class AgentOrchestratorTest { coEvery { settingsRepository.disconnectedApps } returns flowOf(disconnectedApps) coEvery { llmProviderFactory.getProvider(LlmProviderName.GEMINI) } returns llmProvider } + + @Test + fun `observeAndProcessMessages extracts attachments when tool returns imageUri and mimeType`() = + runTest { + val threadId = "thread_1" + val message = createUserMessage(threadId, "generate image of a dog") + val thread = createThread(threadId) + val llmProvider = mockk() + + val generateTool = createMockTool("com.example.appfunctions.agent", "generateImage") + mockAppFunctions(listOf(generateTool)) + setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) + + val toolCall = + LlmResponsePart.ToolCall( + packageName = "com.example.appfunctions.agent", + functionId = "generateImage", + arguments = mapOf("prompt" to "dog"), + callId = "call_1" + ) + + val firstResponse = + LlmResponse.Success( + interactionId = "interaction_1", + parts = listOf(toolCall) + ) + val secondResponse = + LlmResponse.Success( + interactionId = "interaction_2", + parts = listOf(LlmResponsePart.Text("Here is your image!")) + ) + + coEvery { + llmProvider.generateResponse( + previousInteractionId = null, + input = any(), + tools = any(), + apiKey = any(), + modelName = any() + ) + } returns firstResponse + + coEvery { + llmProvider.generateResponse( + previousInteractionId = "interaction_1", + input = any(), + tools = any(), + apiKey = any(), + modelName = any() + ) + } returns secondResponse + + coEvery { + convertInputToAppFunctionDataUseCase(any(), any(), any()) + } returns Result.success(AppFunctionData.EMPTY) + + coEvery { + executeAppFunctionUseCase(any(), any(), any()) + } returns + ExecuteAppFunctionResult.Data( + data = AppFunctionData.EMPTY, + formattedJson = """{"imageUri":"content://com.example.appfunctions.agent.fileprovider/cache/test.jpg","mimeType":"image/jpeg","prompt":"dog"}""" + ) + + agentOrchestrator.observeAndProcessMessages(threadId) + + coVerify { + sendMessageUseCase( + threadId = threadId, + role = MessageRole.ASSISTANT, + textContent = "Here is your image!", + processingStatus = MessageProcessingStatus.PROCESSED, + pendingIntentId = null, + targetPackageName = null, + attachments = + listOf( + com.example.appfunctions.agent.data.db.entities.MessageAttachment( + uri = "content://com.example.appfunctions.agent.fileprovider/cache/test.jpg", + mimeType = "image/jpeg" + ) + ) + ) + } + } } diff --git a/agent/gradle/libs.versions.toml b/agent/gradle/libs.versions.toml index f363f9a..c71351d 100644 --- a/agent/gradle/libs.versions.toml +++ b/agent/gradle/libs.versions.toml @@ -92,6 +92,7 @@ spotless = { id = "com.diffplug.spotless", version.ref = "spotless" } ksp = { id = "com.google.devtools.ksp", version.ref = "ksp" } screenshot = { id = "com.android.compose.screenshot", version.ref = "screenshot" } hilt = { id = "com.google.dagger.hilt.android", version.ref = "hilt" } +kotlin-serialization = { id = "org.jetbrains.kotlin.plugin.serialization", version.ref = "kotlin" } oss-licenses = { id = "com.google.android.gms.oss-licenses-plugin", version.ref = "ossLicensesPlugin" } From 81197ad716e800aa192bdfafd8c4eb89b0ab14f3 Mon Sep 17 00:00:00 2001 From: Caleb Areeveso Date: Wed, 8 Jul 2026 21:17:45 +0000 Subject: [PATCH 02/16] fix: address gemini-code-assist bot code review feedback Change-Id: Ie3fbb88ae72223cc9b1fb29ca226fdc045200104 --- .../agent/data/BuiltInAppFunctions.kt | 22 +++++- .../agent/domain/AgentOrchestrator.kt | 18 +++++ .../agent/domain/AgentOrchestratorTest.kt | 75 +++++++++++++++++++ 3 files changed, 113 insertions(+), 2 deletions(-) diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/BuiltInAppFunctions.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/BuiltInAppFunctions.kt index 08ea703..5b57c14 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/BuiltInAppFunctions.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/BuiltInAppFunctions.kt @@ -203,6 +203,20 @@ class BuiltInAppFunctions { ) } ) + put( + "generationConfig", + JSONObject().apply { + put("responseModalities", JSONArray().apply { put("IMAGE") }) + if (!aspectRatio.isNullOrBlank()) { + put( + "imageConfig", + JSONObject().apply { + put("aspectRatio", aspectRatio) + } + ) + } + } + ) } val endpointUrl = @@ -247,7 +261,9 @@ class BuiltInAppFunctions { .optJSONObject("content") ?.optJSONArray("parts") if (parts == null || parts.length() == 0) { - throw IllegalStateException("No parts returned in candidate content") + throw IllegalStateException( + "No parts returned in candidate content. Gemini response: $responseText" + ) } var base64Data: String? = null @@ -272,7 +288,9 @@ class BuiltInAppFunctions { } if (base64Data.isNullOrBlank()) { - throw IllegalStateException("No inlineData image found in response parts") + throw IllegalStateException( + "No inlineData image found in response parts. Gemini response: $responseText" + ) } val imageBytes = Base64.decode(base64Data, Base64.DEFAULT) diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt index a426cdc..92dc4ce 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt @@ -15,6 +15,9 @@ */ package com.example.appfunctions.agent.domain +import android.content.Context +import android.content.Intent +import android.net.Uri import android.util.Log import androidx.appfunctions.metadata.AppFunctionMetadata import com.example.appfunctions.agent.data.LlmProviderName @@ -36,6 +39,7 @@ import com.example.appfunctions.agent.domain.chat.UpdateMessageUseCase import com.example.appfunctions.agent.domain.chat.UpdateThreadParams import com.example.appfunctions.agent.domain.chat.UpdateThreadUseCase import com.example.appfunctions.agent.domain.pendingintent.SavePendingIntentUseCase +import dagger.hilt.android.qualifiers.ApplicationContext import javax.inject.Inject import javax.inject.Singleton import kotlinx.coroutines.Dispatchers @@ -55,6 +59,7 @@ import org.json.JSONObject class AgentOrchestrator @Inject constructor( + @ApplicationContext private val context: Context, private val manageThreadsUseCase: ManageThreadsUseCase, private val observePendingMessagesUseCase: ObservePendingMessagesUseCase, private val sendMessageUseCase: SendMessageUseCase, @@ -352,6 +357,19 @@ constructor( val convertedInputs = toolCall.arguments.filterValues { it != null } as Map + for (value in convertedInputs.values) { + if (value is String && value.startsWith("content://")) { + runCatching { + val uri = Uri.parse(value) + context.grantUriPermission( + toolCall.packageName, + uri, + Intent.FLAG_GRANT_READ_URI_PERMISSION + ) + } + } + } + val appFunctionDataResult = withContext(Dispatchers.Default) { convertInputToAppFunctionDataUseCase( diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt index 5fbe62e..cedc980 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt @@ -15,6 +15,7 @@ */ package com.example.appfunctions.agent.domain +import android.content.Intent import androidx.appfunctions.AppFunctionData import androidx.appfunctions.metadata.AppFunctionMetadata import androidx.appfunctions.metadata.AppFunctionPackageMetadata @@ -65,6 +66,7 @@ class AgentOrchestratorTest { private val executeAppFunctionUseCase: ExecuteAppFunctionUseCase = mockk() private val sendMessageUseCase: SendMessageUseCase = mockk(relaxed = true) private val convertInputToAppFunctionDataUseCase: ConvertInputToAppFunctionDataUseCase = mockk() + private val context: android.content.Context = mockk(relaxed = true) private val savePendingIntentUseCase: SavePendingIntentUseCase = mockk(relaxed = true) private lateinit var agentOrchestrator: AgentOrchestrator @@ -73,6 +75,7 @@ class AgentOrchestratorTest { fun setUp() { agentOrchestrator = AgentOrchestrator( + context = context, manageThreadsUseCase = manageThreadsUseCase, observePendingMessagesUseCase = observePendingMessagesUseCase, sendMessageUseCase = sendMessageUseCase, @@ -400,4 +403,76 @@ class AgentOrchestratorTest { ) } } + + @Test + fun `observeAndProcessMessages grants URI read permission when tool is called with content URI argument`() = + runTest { + val threadId = "thread_1" + val message = createUserMessage(threadId, "set wallpaper") + val thread = createThread(threadId) + val llmProvider = mockk() + + val targetTool = createMockTool("com.example.targetapp", "setWallpaper") + mockAppFunctions(listOf(targetTool)) + setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) + + val contentUri = "content://com.example.appfunctions.agent.fileprovider/cache/img.jpg" + val toolCall = + LlmResponsePart.ToolCall( + packageName = "com.example.targetapp", + functionId = "setWallpaper", + arguments = mapOf("uri" to contentUri), + callId = "call_2" + ) + + coEvery { + llmProvider.generateResponse( + previousInteractionId = null, + input = any(), + tools = any(), + apiKey = any(), + modelName = any() + ) + } returns + LlmResponse.Success( + interactionId = "interaction_1", + parts = listOf(toolCall) + ) + + coEvery { + llmProvider.generateResponse( + previousInteractionId = "interaction_1", + input = any(), + tools = any(), + apiKey = any(), + modelName = any() + ) + } returns + LlmResponse.Success( + interactionId = "interaction_2", + parts = listOf(LlmResponsePart.Text("Wallpaper set")) + ) + + coEvery { + convertInputToAppFunctionDataUseCase(any(), any(), any()) + } returns Result.success(AppFunctionData.EMPTY) + + coEvery { + executeAppFunctionUseCase(any(), any(), any()) + } returns + ExecuteAppFunctionResult.Data( + data = AppFunctionData.EMPTY, + formattedJson = """{"success":true}""" + ) + + agentOrchestrator.observeAndProcessMessages(threadId) + + coVerify { + context.grantUriPermission( + eq("com.example.targetapp"), + any(), + eq(Intent.FLAG_GRANT_READ_URI_PERMISSION) + ) + } + } } From eccfb9b653a51060b8146c60fe0cd262607527f0 Mon Sep 17 00:00:00 2001 From: Caleb Areeveso Date: Thu, 9 Jul 2026 15:10:25 +0000 Subject: [PATCH 03/16] agent & ChatApp: Integrate cross-app AppFunctions and media attachments Change-Id: Id88dc93971c3c21a8044d125070785dd3c879fb9 --- .../chatapp/appfunctions/AppFunctionsTest.kt | 13 +- .../kotlin/com/example/chatapp/InputBar.kt | 13 +- .../chatapp/uicomponents/ChatScreen.kt | 185 ++++++++++-------- ChatApp/gradle/gradle-daemon-jvm.properties | 12 ++ .../com/example/chatapp/ChatViewModel.kt | 9 + .../chatapp/appfunctions/AppFunctions.kt | 28 +++ .../chatapp/data/WallpaperRepository.kt | 81 ++++++++ .../agent/data/GeminiProviderImpl.kt | 6 +- .../agent/data/GeminiToolConverter.kt | 83 +++++--- .../agent/domain/AgentOrchestrator.kt | 182 +++++++++++++++-- .../ConvertInputToAppFunctionDataUseCase.kt | 26 ++- .../agent/data/GeminiToolConverterTest.kt | 112 +++++++++++ ...onvertInputToAppFunctionDataUseCaseTest.kt | 28 +++ 13 files changed, 652 insertions(+), 126 deletions(-) create mode 100644 ChatApp/gradle/gradle-daemon-jvm.properties create mode 100644 ChatApp/shared/src/main/kotlin/com/example/chatapp/data/WallpaperRepository.kt diff --git a/ChatApp/app/src/androidTest/kotlin/com/example/chatapp/appfunctions/AppFunctionsTest.kt b/ChatApp/app/src/androidTest/kotlin/com/example/chatapp/appfunctions/AppFunctionsTest.kt index d4095e3..a87f656 100644 --- a/ChatApp/app/src/androidTest/kotlin/com/example/chatapp/appfunctions/AppFunctionsTest.kt +++ b/ChatApp/app/src/androidTest/kotlin/com/example/chatapp/appfunctions/AppFunctionsTest.kt @@ -27,8 +27,11 @@ import com.example.chatapp.data.CallManager import com.example.chatapp.data.DisplayMessage import com.example.chatapp.data.MessageRepository import com.example.chatapp.data.RecipientsRepository +import com.example.chatapp.data.WallpaperRepository +import java.io.InputStream import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.flowOf import kotlinx.coroutines.flow.map import kotlinx.coroutines.runBlocking import kotlinx.coroutines.test.runTest @@ -85,7 +88,15 @@ class AppFunctionsTest { private val callManager = CallManager(recipientsRepository) - private val appFunctions = AppFunctions(messageRepository, recipientsRepository, callManager) + private val mockWallpaperRepository = + object : WallpaperRepository { + override fun getWallpaper(chatId: String): Flow = flowOf(null) + + override suspend fun setWallpaper(chatId: String, inputStream: InputStream): Boolean = true + } + + private val appFunctions = + AppFunctions(messageRepository, recipientsRepository, callManager, mockWallpaperRepository) @Test(expected = AppFunctionInvalidArgumentException::class) fun searchContacts_returnsEmptyList() { diff --git a/ChatApp/app/src/main/kotlin/com/example/chatapp/InputBar.kt b/ChatApp/app/src/main/kotlin/com/example/chatapp/InputBar.kt index 2c8c382..3cc9724 100644 --- a/ChatApp/app/src/main/kotlin/com/example/chatapp/InputBar.kt +++ b/ChatApp/app/src/main/kotlin/com/example/chatapp/InputBar.kt @@ -38,6 +38,7 @@ import androidx.compose.material.icons.filled.Add import androidx.compose.material3.FilledIconButton import androidx.compose.material3.Icon import androidx.compose.material3.IconButton +import androidx.compose.material3.IconButtonDefaults import androidx.compose.material3.MaterialTheme import androidx.compose.material3.Surface import androidx.compose.material3.Text @@ -77,7 +78,8 @@ internal fun InputBar( Surface( modifier = modifier, - tonalElevation = 3.dp, + color = Color.Transparent, + tonalElevation = 0.dp, ) { Column { if (selectedImages.isNotEmpty()) { @@ -107,18 +109,25 @@ internal fun InputBar( verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(4.dp), ) { - IconButton( + FilledIconButton( onClick = { photoPickerLauncher.launch( PickVisualMediaRequest(ActivityResultContracts.PickVisualMedia.ImageOnly), ) }, + modifier = Modifier.size(56.dp), + colors = + IconButtonDefaults.filledIconButtonColors( + containerColor = MaterialTheme.colorScheme.surfaceVariant, + contentColor = MaterialTheme.colorScheme.onSurfaceVariant, + ), ) { Icon( imageVector = Icons.Default.Add, contentDescription = "Add Image", ) } + Spacer(modifier = Modifier.width(4.dp)) TextField( value = value, onValueChange = onInputChanged, diff --git a/ChatApp/app/src/main/kotlin/com/example/chatapp/uicomponents/ChatScreen.kt b/ChatApp/app/src/main/kotlin/com/example/chatapp/uicomponents/ChatScreen.kt index 388dd89..a09910e 100644 --- a/ChatApp/app/src/main/kotlin/com/example/chatapp/uicomponents/ChatScreen.kt +++ b/ChatApp/app/src/main/kotlin/com/example/chatapp/uicomponents/ChatScreen.kt @@ -15,7 +15,9 @@ */ package com.example.chatapp.uicomponents +import androidx.compose.foundation.background import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.PaddingValues import androidx.compose.foundation.layout.consumeWindowInsets @@ -57,6 +59,7 @@ import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.draw.clip +import androidx.compose.ui.graphics.Color import androidx.compose.ui.input.nestedscroll.nestedScroll import androidx.compose.ui.layout.ContentScale import androidx.compose.ui.res.stringResource @@ -90,99 +93,123 @@ fun ChatScreen( val uiState by viewModel.uiState.collectAsStateWithLifecycle() var message by rememberSaveable { mutableStateOf("") } - Scaffold( - modifier = - Modifier - .fillMaxSize() - .nestedScroll(scrollBehavior.nestedScrollConnection), - topBar = { - TopAppBar( - colors = - topAppBarColors( - containerColor = MaterialTheme.colorScheme.primaryContainer, - titleContentColor = MaterialTheme.colorScheme.onPrimaryContainer, - ), - title = { - Text(text = viewModel.recipient.name) - }, - navigationIcon = { - IconButton(onClick = onBackClick) { - Icon( - imageVector = Icons.AutoMirrored.Filled.ArrowBack, - contentDescription = "Back", - ) - } - }, - actions = { - IconButton(onClick = onCallClick) { - Icon( - imageVector = Icons.Default.Call, - contentDescription = "Call", - ) - } - }, + Box(modifier = Modifier.fillMaxSize()) { + uiState.wallpaperPath?.let { path -> + AsyncImage( + model = path, + contentDescription = "Chat Wallpaper", + contentScale = ContentScale.Crop, + modifier = Modifier.fillMaxSize(), ) - }, - bottomBar = { - InputBar( - value = message, - placeholder = stringResource(R.string.input_placeholder), - onInputChanged = { - message = it - }, - onSendClick = { uris -> - viewModel.sendMessage(message, uris) - message = "" - }, - sendEnabled = uiState.botMessageState !is BotMessageState.Generating, + Box( modifier = Modifier - .navigationBarsPadding() - .imePadding(), + .fillMaxSize() + .background(Color.Black.copy(alpha = 0.45f)), ) - }, - ) { innerPadding -> - Column( + } + + Scaffold( modifier = Modifier .fillMaxSize() - .padding(innerPadding) - .consumeWindowInsets(innerPadding), - ) { - MessageList( + .nestedScroll(scrollBehavior.nestedScrollConnection), + containerColor = + if (uiState.wallpaperPath != null) { + Color.Transparent + } else { + MaterialTheme.colorScheme.surfaceVariant + }, + topBar = { + TopAppBar( + colors = + topAppBarColors( + containerColor = MaterialTheme.colorScheme.primaryContainer, + titleContentColor = MaterialTheme.colorScheme.onPrimaryContainer, + ), + title = { + Text(text = viewModel.recipient.name) + }, + navigationIcon = { + IconButton(onClick = onBackClick) { + Icon( + imageVector = Icons.AutoMirrored.Filled.ArrowBack, + contentDescription = "Back", + ) + } + }, + actions = { + IconButton(onClick = onCallClick) { + Icon( + imageVector = Icons.Default.Call, + contentDescription = "Call", + ) + } + }, + ) + }, + bottomBar = { + InputBar( + value = message, + placeholder = stringResource(R.string.input_placeholder), + onInputChanged = { + message = it + }, + onSendClick = { uris -> + viewModel.sendMessage(message, uris) + message = "" + }, + sendEnabled = uiState.botMessageState !is BotMessageState.Generating, + modifier = + Modifier + .navigationBarsPadding() + .imePadding(), + ) + }, + ) { innerPadding -> + Box( modifier = Modifier - .fillMaxWidth() - .padding(horizontal = 16.dp) - .weight(1f), - messages = uiState.messages, - contentPadding = PaddingValues(bottom = 8.dp), - ) - - when (val state = uiState.botMessageState) { - is BotMessageState.Generating -> { - CircularProgressIndicator( + .fillMaxSize() + .padding(innerPadding) + .consumeWindowInsets(innerPadding), + ) { + Column(modifier = Modifier.fillMaxSize()) { + MessageList( modifier = Modifier - .padding(vertical = 8.dp) - .align(Alignment.CenterHorizontally), + .fillMaxWidth() + .padding(horizontal = 16.dp) + .weight(1f), + messages = uiState.messages, + contentPadding = PaddingValues(bottom = 8.dp), ) - } - is BotMessageState.Error -> { - AlertDialog( - onDismissRequest = { viewModel.dismissError() }, - title = { Text(text = stringResource(R.string.error)) }, - text = { Text(text = state.errorMessage) }, - confirmButton = { - Button(onClick = { viewModel.dismissError() }) { - Text(text = stringResource(R.string.dismiss_button)) - } - }, - ) - } + when (val state = uiState.botMessageState) { + is BotMessageState.Generating -> { + CircularProgressIndicator( + modifier = + Modifier + .padding(vertical = 8.dp) + .align(Alignment.CenterHorizontally), + ) + } - else -> { // No additional UI for waiting state + is BotMessageState.Error -> { + AlertDialog( + onDismissRequest = { viewModel.dismissError() }, + title = { Text(text = stringResource(R.string.error)) }, + text = { Text(text = state.errorMessage) }, + confirmButton = { + Button(onClick = { viewModel.dismissError() }) { + Text(text = stringResource(R.string.dismiss_button)) + } + }, + ) + } + + else -> {} + } } } } diff --git a/ChatApp/gradle/gradle-daemon-jvm.properties b/ChatApp/gradle/gradle-daemon-jvm.properties new file mode 100644 index 0000000..6c1139e --- /dev/null +++ b/ChatApp/gradle/gradle-daemon-jvm.properties @@ -0,0 +1,12 @@ +#This file is generated by updateDaemonJvm +toolchainUrl.FREE_BSD.AARCH64=https\://api.foojay.io/disco/v3.0/ids/ec7520a1e057cd116f9544c42142a16b/redirect +toolchainUrl.FREE_BSD.X86_64=https\://api.foojay.io/disco/v3.0/ids/4c4f879899012ff0a8b2e2117df03b0e/redirect +toolchainUrl.LINUX.AARCH64=https\://api.foojay.io/disco/v3.0/ids/ec7520a1e057cd116f9544c42142a16b/redirect +toolchainUrl.LINUX.X86_64=https\://api.foojay.io/disco/v3.0/ids/4c4f879899012ff0a8b2e2117df03b0e/redirect +toolchainUrl.MAC_OS.AARCH64=https\://api.foojay.io/disco/v3.0/ids/73bcfb608d1fde9fb62e462f834a3299/redirect +toolchainUrl.MAC_OS.X86_64=https\://api.foojay.io/disco/v3.0/ids/846ee0d876d26a26f37aa1ce8de73224/redirect +toolchainUrl.UNIX.AARCH64=https\://api.foojay.io/disco/v3.0/ids/ec7520a1e057cd116f9544c42142a16b/redirect +toolchainUrl.UNIX.X86_64=https\://api.foojay.io/disco/v3.0/ids/4c4f879899012ff0a8b2e2117df03b0e/redirect +toolchainUrl.WINDOWS.AARCH64=https\://api.foojay.io/disco/v3.0/ids/9482ddec596298c84656d31d16652665/redirect +toolchainUrl.WINDOWS.X86_64=https\://api.foojay.io/disco/v3.0/ids/39701d92e1756bb2f141eb67cd4c660e/redirect +toolchainVersion=21 diff --git a/ChatApp/shared/src/main/kotlin/com/example/chatapp/ChatViewModel.kt b/ChatApp/shared/src/main/kotlin/com/example/chatapp/ChatViewModel.kt index a7ec427..6144ab9 100644 --- a/ChatApp/shared/src/main/kotlin/com/example/chatapp/ChatViewModel.kt +++ b/ChatApp/shared/src/main/kotlin/com/example/chatapp/ChatViewModel.kt @@ -23,6 +23,7 @@ import com.example.chatapp.data.CallManager import com.example.chatapp.data.DisplayMessage import com.example.chatapp.data.MessageRepository import com.example.chatapp.data.RecipientsRepository +import com.example.chatapp.data.WallpaperRepository import dagger.assisted.Assisted import dagger.assisted.AssistedFactory import dagger.assisted.AssistedInject @@ -64,6 +65,8 @@ data class ChatbotUiState( val messages: List = listOf(), /** The current state of the bot's response generation. */ val botMessageState: BotMessageState = BotMessageState.WaitingForMessage, + /** Optional path to a custom wallpaper image for this chat. */ + val wallpaperPath: String? = null, ) @HiltViewModel(assistedFactory = ChatViewModel.Factory::class) @@ -74,6 +77,7 @@ class ChatViewModel private val messageRepository: MessageRepository, private val callManager: CallManager, private val recipientsRepository: RecipientsRepository, + private val wallpaperRepository: WallpaperRepository, ) : ViewModel() { @AssistedFactory interface Factory { @@ -102,6 +106,11 @@ class ChatViewModel _uiState.update { it.copy(messages = msgs) } } } + viewModelScope.launch { + wallpaperRepository.getWallpaper(recipientId).collect { path -> + _uiState.update { it.copy(wallpaperPath = path) } + } + } } fun startCall() { diff --git a/ChatApp/shared/src/main/kotlin/com/example/chatapp/appfunctions/AppFunctions.kt b/ChatApp/shared/src/main/kotlin/com/example/chatapp/appfunctions/AppFunctions.kt index 81a6574..e12ed01 100644 --- a/ChatApp/shared/src/main/kotlin/com/example/chatapp/appfunctions/AppFunctions.kt +++ b/ChatApp/shared/src/main/kotlin/com/example/chatapp/appfunctions/AppFunctions.kt @@ -28,6 +28,7 @@ import androidx.appfunctions.service.AppFunction import com.example.chatapp.data.CallManager import com.example.chatapp.data.MessageRepository import com.example.chatapp.data.RecipientsRepository +import com.example.chatapp.data.WallpaperRepository import javax.inject.Inject /** @@ -39,6 +40,7 @@ class AppFunctions private val messageRepository: MessageRepository, private val recipientsRepository: RecipientsRepository, private val callManager: CallManager, + private val wallpaperRepository: WallpaperRepository, ) { /** * Search for contacts or groups by name. @@ -165,6 +167,32 @@ class AppFunctions ) } + /** + * Updates the wallpaper image for a specific chat conversation. + * + * @param appFunctionContext The context of this app function call. + * @param chatId The unique identifier for the recipient or chat group. + * @param wallpaperUri The URI of the image file to set as the chat wallpaper. + */ + @AppFunction(isDescribedByKDoc = true) + suspend fun updateChatWallpaper( + appFunctionContext: AppFunctionContext, + chatId: String, + wallpaperUri: Uri, + ): Boolean { + val resolvedId = + recipientsRepository.getRecipientById(chatId)?.id + ?: recipientsRepository.getGroupById(chatId)?.id + ?: recipientsRepository.searchAny(chatId, maxCount = 1).firstOrNull()?.endpointValue + ?: chatId + val inputStream = + appFunctionContext.context.contentResolver.openInputStream(wallpaperUri) + ?: throw AppFunctionInvalidArgumentException("Cannot open wallpaper stream") + return inputStream.use { stream -> + wallpaperRepository.setWallpaper(resolvedId, stream) + } + } + /** Represents a result from a contact or group search. */ @AppFunctionSerializable(isDescribedByKDoc = true) data class ContactSearchResult( diff --git a/ChatApp/shared/src/main/kotlin/com/example/chatapp/data/WallpaperRepository.kt b/ChatApp/shared/src/main/kotlin/com/example/chatapp/data/WallpaperRepository.kt new file mode 100644 index 0000000..0122f2f --- /dev/null +++ b/ChatApp/shared/src/main/kotlin/com/example/chatapp/data/WallpaperRepository.kt @@ -0,0 +1,81 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.chatapp.data + +import android.content.Context +import dagger.Binds +import dagger.Module +import dagger.hilt.InstallIn +import dagger.hilt.android.qualifiers.ApplicationContext +import dagger.hilt.components.SingletonComponent +import java.io.File +import java.io.InputStream +import javax.inject.Inject +import javax.inject.Singleton +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.update +import kotlinx.coroutines.withContext + +interface WallpaperRepository { + fun getWallpaper(chatId: String): Flow + suspend fun setWallpaper(chatId: String, inputStream: InputStream): Boolean +} + +@Singleton +class WallpaperRepositoryImpl @Inject constructor( + @ApplicationContext private val context: Context, +) : WallpaperRepository { + private val wallpapers = MutableStateFlow>(emptyMap()) + + override fun getWallpaper(chatId: String): Flow { + return wallpapers.map { map -> + map[chatId] ?: run { + val dir = File(context.filesDir, "wallpapers") + dir.listFiles { f -> f.name.startsWith("wallpaper_${chatId}_") } + ?.maxByOrNull { it.lastModified() } + ?.absolutePath + } + } + } + + override suspend fun setWallpaper( + chatId: String, + inputStream: InputStream, + ): Boolean = withContext(Dispatchers.IO) { + try { + val dir = File(context.filesDir, "wallpapers").apply { mkdirs() } + dir.listFiles { f -> f.name.startsWith("wallpaper_${chatId}_") }?.forEach { it.delete() } + val file = File(dir, "wallpaper_${chatId}_${System.currentTimeMillis()}.jpg") + file.outputStream().use { output -> + inputStream.copyTo(output) + } + wallpapers.update { current -> current + (chatId to file.absolutePath) } + true + } catch (e: Exception) { + false + } + } +} + +@Module +@InstallIn(SingletonComponent::class) +abstract class WallpaperModule { + @Binds + abstract fun bindWallpaperRepository(impl: WallpaperRepositoryImpl): WallpaperRepository +} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt index e061216..ceef358 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt @@ -55,10 +55,12 @@ constructor( input: LlmInput, tools: List, apiKey: String, - modelName: String + modelName: String, ): LlmResponse { + val sortedTools = tools.sortedByDescending { it.id.startsWith(it.packageName) } + val uniqueTools = sortedTools.distinctBy { toolConverter.getToolName(it) } val convertedTools = - tools.mapNotNull { tool -> + uniqueTools.mapNotNull { tool -> try { buildJsonObject { put(KEY_TYPE, JsonPrimitive(VALUE_FUNCTION)) diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt index 7de144e..35cbee1 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt @@ -67,6 +67,7 @@ class GeminiToolConverter parameter.dataType, tool.components, tool.id, + parameterName = parameter.name, ) put( parameter.name, @@ -118,11 +119,15 @@ class GeminiToolConverter components: AppFunctionComponentsMetadata, functionId: String, visitedReferences: Set = emptySet(), + parameterName: String? = null, ): JsonObject { return when (dataType) { is AppFunctionStringTypeMetadata -> buildJsonObject { put(KEY_TYPE, JsonPrimitive(VALUE_STRING)) + if (isFileReferenceParameter(parameterName)) { + put(KEY_FORMAT, JsonPrimitive(VALUE_FILE_REFERENCE)) + } val enumValues = dataType.enumValues if (!enumValues.isNullOrEmpty()) { put( @@ -168,39 +173,54 @@ class GeminiToolConverter components, functionId, visitedReferences, + parameterName, ), ) } is AppFunctionObjectTypeMetadata -> - buildJsonObject { - put(KEY_TYPE, JsonPrimitive(VALUE_OBJECT)) - put( - KEY_PROPERTIES, - buildJsonObject { - dataType.properties.forEach { (name, type) -> - put( - name, - mapDataTypeToGeminiSchema( - type, - components, - functionId, - visitedReferences, - ), - ) - } - }, - ) - if (dataType.required.isNotEmpty()) { + if (dataType.qualifiedName == "android.net.Uri") { + buildJsonObject { + put(KEY_TYPE, JsonPrimitive(VALUE_STRING)) + put(KEY_FORMAT, JsonPrimitive(VALUE_FILE_REFERENCE)) + } + } else { + buildJsonObject { + put(KEY_TYPE, JsonPrimitive(VALUE_OBJECT)) put( - KEY_REQUIRED, - buildJsonArray { - dataType.required.forEach { name -> add(JsonPrimitive(name)) } + KEY_PROPERTIES, + buildJsonObject { + dataType.properties.forEach { (name, type) -> + put( + name, + mapDataTypeToGeminiSchema( + type, + components, + functionId, + visitedReferences, + parameterName = name, + ), + ) + } }, ) + if (dataType.required.isNotEmpty()) { + put( + KEY_REQUIRED, + buildJsonArray { + dataType.required.forEach { name -> add(JsonPrimitive(name)) } + }, + ) + } } } is AppFunctionReferenceTypeMetadata -> { val referenceKey = dataType.referenceDataType + if (referenceKey == "android.net.Uri") { + return buildJsonObject { + put(KEY_TYPE, JsonPrimitive(VALUE_STRING)) + put(KEY_FORMAT, JsonPrimitive(VALUE_FILE_REFERENCE)) + } + } if (visitedReferences.contains(referenceKey)) { Log.d( "GeminiToolConverter", @@ -218,6 +238,7 @@ class GeminiToolConverter components, functionId, visitedReferences + referenceKey, + parameterName, ) } else -> @@ -227,12 +248,30 @@ class GeminiToolConverter } } + private fun isFileReferenceParameter(parameterName: String?): Boolean { + if (parameterName == null) return false + if (parameterName in KNOWN_FILE_REFERENCE_PARAM_NAMES) return true + return parameterName.endsWith("Uri", ignoreCase = true) || + parameterName.endsWith("Uris", ignoreCase = true) + } + companion object { private const val TOOL_ID_SEPARATOR = "_" private const val KEY_NAME = "name" private const val KEY_DESCRIPTION = "description" private const val KEY_PARAMETERS = "parameters" private const val KEY_TYPE = "type" + private const val KEY_FORMAT = "format" + private const val VALUE_FILE_REFERENCE = "file_reference" + private val KNOWN_FILE_REFERENCE_PARAM_NAMES = + setOf( + "wallpaperUri", + "imageUri", + "attachmentUri", + "ringtoneUri", + "profilePictureUri", + "audioUri", + ) private const val VALUE_OBJECT = "object" private const val KEY_PROPERTIES = "properties" private const val KEY_REQUIRED = "required" diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt index 92dc4ce..092618f 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt @@ -19,7 +19,17 @@ import android.content.Context import android.content.Intent import android.net.Uri import android.util.Log +import androidx.appfunctions.metadata.AppFunctionArrayTypeMetadata +import androidx.appfunctions.metadata.AppFunctionDataTypeMetadata import androidx.appfunctions.metadata.AppFunctionMetadata +import androidx.appfunctions.metadata.AppFunctionObjectTypeMetadata +import androidx.appfunctions.metadata.AppFunctionParameterMetadata +import androidx.appfunctions.metadata.AppFunctionReferenceTypeMetadata +import androidx.core.content.FileProvider +import java.io.File +import java.net.HttpURLConnection +import java.net.URL +import java.util.UUID import com.example.appfunctions.agent.data.LlmProviderName import com.example.appfunctions.agent.data.SettingsRepository import com.example.appfunctions.agent.data.db.entities.MessageAttachment @@ -144,11 +154,14 @@ constructor( disconnectedApps: Set, targetPackageName: String? ): List { - return allTools.filter { metadata -> - metadata.isEnabled && - metadata.packageName !in disconnectedApps && - (targetPackageName == null || metadata.packageName == targetPackageName) - } + return allTools + .filter { metadata -> + metadata.isEnabled && + metadata.packageName !in disconnectedApps && + (targetPackageName == null || metadata.packageName == targetPackageName) + } + .sortedByDescending { metadata -> metadata.id.startsWith(metadata.packageName) } + .distinctBy { metadata -> metadata.id } } private suspend fun runInteractionLoop( @@ -355,20 +368,26 @@ constructor( return ExecuteToolCallsResult.Error } - val convertedInputs = toolCall.arguments.filterValues { it != null } as Map - - for (value in convertedInputs.values) { - if (value is String && value.startsWith("content://")) { - runCatching { - val uri = Uri.parse(value) - context.grantUriPermission( - toolCall.packageName, - uri, - Intent.FLAG_GRANT_READ_URI_PERMISSION + val rawConvertedInputs = toolCall.arguments.filterValues { it != null } as Map + val convertedInputs = + try { + withContext(Dispatchers.IO) { + resolveRemoteFileReferencesRecursively( + context = context, + parametersMetadata = matchingTool.parameters, + inputs = rawConvertedInputs, ) } + } catch (e: Exception) { + completeMessageWithError( + message.messageId, + message.threadId, + "Failed to download remote file reference: ${e.message}", + ) + return ExecuteToolCallsResult.Error } - } + + grantContentUriPermissionsRecursively(context, toolCall.packageName, convertedInputs) val appFunctionDataResult = withContext(Dispatchers.Default) { @@ -440,4 +459,135 @@ constructor( processingStatus = MessageProcessingStatus.FAILED ) } + + private fun resolveRemoteFileReferencesRecursively( + context: Context, + parametersMetadata: List, + inputs: Map, + ): Map { + val paramMap = parametersMetadata.associateBy { it.name } + return inputs.mapValues { (key, value) -> + val paramMeta = paramMap[key] + resolveValueRecursively(context, paramMeta?.dataType, value, key) + } + } + + private fun resolveValueRecursively( + context: Context, + dataType: AppFunctionDataTypeMetadata?, + value: Any, + paramName: String?, + ): Any { + return when (value) { + is String -> { + val shouldResolve = + isFileReferenceParameter(paramName) || isUriMetadata(dataType) + if (shouldResolve && (value.startsWith("http://") || value.startsWith("https://"))) { + downloadRemoteFileToContentUri(context, value) + } else { + value + } + } + is Map<*, *> -> { + value.entries.associate { entry -> + val k = entry.key as String + val propType = + (dataType as? AppFunctionObjectTypeMetadata)?.properties?.get(k) + k to (entry.value?.let { resolveValueRecursively(context, propType, it, k) } ?: "") + } + } + is List<*> -> { + val itemType = (dataType as? AppFunctionArrayTypeMetadata)?.itemType + value.mapNotNull { item -> + item?.let { resolveValueRecursively(context, itemType, it, paramName) } + } + } + else -> value + } + } + + private fun downloadRemoteFileToContentUri(context: Context, urlString: String): String { + val url = URL(urlString) + val connection = url.openConnection() as HttpURLConnection + try { + connection.connect() + val contentType = connection.contentType ?: "" + val ext = + when { + contentType.contains("png", ignoreCase = true) -> "png" + contentType.contains("jpeg", ignoreCase = true) || + contentType.contains("jpg", ignoreCase = true) -> "jpg" + contentType.contains("gif", ignoreCase = true) -> "gif" + contentType.contains("webp", ignoreCase = true) -> "webp" + urlString.substringAfterLast("/", "").contains(".") -> + urlString.substringAfterLast("/").substringAfterLast(".") + else -> "jpg" + } + val cacheDir = File(context.cacheDir, "file_references").apply { mkdirs() } + val file = File(cacheDir, "generated_${UUID.randomUUID()}.$ext") + connection.inputStream.use { input -> + file.outputStream().use { output -> + input.copyTo(output) + } + } + val contentUri = + FileProvider.getUriForFile( + context, + "${context.packageName}.fileprovider", + file, + ) + return contentUri.toString() + } finally { + connection.disconnect() + } + } + + private fun grantContentUriPermissionsRecursively( + context: Context, + targetPackageName: String, + value: Any?, + ) { + when (value) { + is String -> { + if (value.startsWith("content://")) { + runCatching { + val uri = Uri.parse(value) + context.grantUriPermission( + targetPackageName, + uri, + Intent.FLAG_GRANT_READ_URI_PERMISSION, + ) + } + } + } + is Map<*, *> -> value.values.forEach { grantContentUriPermissionsRecursively(context, targetPackageName, it) } + is List<*> -> value.forEach { grantContentUriPermissionsRecursively(context, targetPackageName, it) } + } + } + + private fun isFileReferenceParameter(parameterName: String?): Boolean { + if (parameterName == null) return false + if (parameterName in KNOWN_FILE_REFERENCE_PARAM_NAMES) return true + return parameterName.endsWith("Uri", ignoreCase = true) || + parameterName.endsWith("Uris", ignoreCase = true) + } + + private fun isUriMetadata(dataType: AppFunctionDataTypeMetadata?): Boolean { + if (dataType == null) return false + if (dataType is AppFunctionObjectTypeMetadata && dataType.qualifiedName == "android.net.Uri") return true + if (dataType is AppFunctionReferenceTypeMetadata && dataType.referenceDataType == "android.net.Uri") return true + return false + } + + companion object { + private val KNOWN_FILE_REFERENCE_PARAM_NAMES = + setOf( + "wallpaperUri", + "imageUri", + "attachmentUri", + "ringtoneUri", + "profilePictureUri", + "audioUri", + ) + } } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCase.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCase.kt index 7fa27e7..799331a 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCase.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCase.kt @@ -98,8 +98,17 @@ class ConvertInputToAppFunctionDataUseCase } } is AppFunctionObjectTypeMetadata -> { - val objData = convertObject(dataType, value as Map, components) - builder.setAppFunctionData(name, objData) + if (dataType.qualifiedName == "android.net.Uri" && value is String) { + val uriPropertyName = dataType.properties.keys.firstOrNull() ?: "uri" + val uriData = + AppFunctionData.Builder(dataType, components) + .setString(uriPropertyName, value) + .build() + builder.setAppFunctionData(name, uriData) + } else { + val objData = convertObject(dataType, value as Map, components) + builder.setAppFunctionData(name, objData) + } } is AppFunctionArrayTypeMetadata -> { setArrayValue(builder, name, dataType, value as List, components) @@ -109,8 +118,17 @@ class ConvertInputToAppFunctionDataUseCase val objectType = components.dataTypes[referenceKey] as? AppFunctionObjectTypeMetadata if (objectType != null) { - val objData = convertObject(objectType, value as Map, components) - builder.setAppFunctionData(name, objData) + if (referenceKey == "android.net.Uri" && value is String) { + val uriPropertyName = objectType.properties.keys.firstOrNull() ?: "uri" + val uriData = + AppFunctionData.Builder(objectType, components) + .setString(uriPropertyName, value) + .build() + builder.setAppFunctionData(name, uriData) + } else { + val objData = convertObject(objectType, value as Map, components) + builder.setAppFunctionData(name, objData) + } } } } diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiToolConverterTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiToolConverterTest.kt index 2816960..62cb16a 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiToolConverterTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiToolConverterTest.kt @@ -408,4 +408,116 @@ class GeminiToolConverterTest { assertEquals(expectedJson, schema) } + + @Test + fun convert_stringParameterEndingWithUri_injectsFileReferenceFormat() { + val parameter = + AppFunctionParameterMetadata( + name = "wallpaperUri", + isRequired = true, + dataType = AppFunctionStringTypeMetadata(isNullable = false), + description = "A URI parameter", + ) + val tool = + AppFunctionMetadata( + id = "com.example.my_function", + packageName = "com.example", + isEnabled = true, + schema = null, + parameters = listOf(parameter), + response = + AppFunctionResponseMetadata( + valueType = AppFunctionStringTypeMetadata(isNullable = false), + description = "", + ), + components = AppFunctionComponentsMetadata(emptyMap()), + description = "Test function", + deprecation = null, + ) + + val schema = converter.convert(tool) + + val expectedJson = + Json.parseToJsonElement( + """ + { + "name": "com_example_my_function", + "description": "Test function", + "parameters": { + "type": "object", + "properties": { + "wallpaperUri": { + "type": "string", + "format": "file_reference", + "description": "A URI parameter" + } + }, + "required": ["wallpaperUri"] + } + } + """, + ) + + assertEquals(expectedJson, schema) + } + + @Test + fun convert_uriObjectType_injectsFileReferenceFormat() { + val uriObjectMetadata = + AppFunctionObjectTypeMetadata( + properties = emptyMap(), + required = emptyList(), + qualifiedName = "android.net.Uri", + isNullable = false, + description = "Uri object", + ) + val parameter = + AppFunctionParameterMetadata( + name = "customUri", + isRequired = true, + dataType = uriObjectMetadata, + description = "A Uri object parameter", + ) + val tool = + AppFunctionMetadata( + id = "com.example.my_function", + packageName = "com.example", + isEnabled = true, + schema = null, + parameters = listOf(parameter), + response = + AppFunctionResponseMetadata( + valueType = AppFunctionStringTypeMetadata(isNullable = false), + description = "", + ), + components = AppFunctionComponentsMetadata(emptyMap()), + description = "Test function", + deprecation = null, + ) + + val schema = converter.convert(tool) + + val expectedJson = + Json.parseToJsonElement( + """ + { + "name": "com_example_my_function", + "description": "Test function", + "parameters": { + "type": "object", + "properties": { + "customUri": { + "type": "string", + "format": "file_reference", + "description": "A Uri object parameter" + } + }, + "required": ["customUri"] + } + } + """, + ) + + assertEquals(expectedJson, schema) + } } diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCaseTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCaseTest.kt index 25760fb..478030b 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCaseTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCaseTest.kt @@ -238,4 +238,32 @@ class ConvertInputToAppFunctionDataUseCaseTest { assertEquals(true, result.isFailure) } + + @Test + fun convert_uriObjectTypeWithStringInput_buildsUriAppFunctionData() { + val uriObjectType = + AppFunctionObjectTypeMetadata( + properties = + mapOf( + "uri" to AppFunctionStringTypeMetadata(false), + ), + required = listOf("uri"), + qualifiedName = "android.net.Uri", + isNullable = false, + ) + val parameters = + listOf( + AppFunctionParameterMetadata( + name = "wallpaperUri", + isRequired = true, + dataType = uriObjectType, + ), + ) + val inputs = mapOf("wallpaperUri" to "content://com.example/file.jpg") + + val result = useCase(parameters, components, inputs).getOrThrow() + + val uriData = result.getAppFunctionData("wallpaperUri") + assertEquals("content://com.example/file.jpg", uriData?.getString("uri")) + } } From 4bb9b27263ab02ffc982abe18465da2ef4357885 Mon Sep 17 00:00:00 2001 From: Caleb Areeveso Date: Fri, 10 Jul 2026 10:45:08 +0000 Subject: [PATCH 04/16] fix: add sportless formatting Change-Id: I5f3f8d62d3be770e5035fc1c785b346ca74695e1 --- .../agent/data/GeminiProviderImpl.kt | 2 +- .../agent/data/GeminiToolConverter.kt | 445 +++++++++--------- .../agent/domain/AgentOrchestrator.kt | 65 ++- 3 files changed, 271 insertions(+), 241 deletions(-) diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt index ceef358..177c237 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt @@ -55,7 +55,7 @@ constructor( input: LlmInput, tools: List, apiKey: String, - modelName: String, + modelName: String ): LlmResponse { val sortedTools = tools.sortedByDescending { it.id.startsWith(it.packageName) } val uniqueTools = sortedTools.distinctBy { toolConverter.getToolName(it) } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt index 35cbee1..1dda085 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt @@ -29,260 +29,273 @@ import androidx.appfunctions.metadata.AppFunctionObjectTypeMetadata import androidx.appfunctions.metadata.AppFunctionReferenceTypeMetadata import androidx.appfunctions.metadata.AppFunctionStringTypeMetadata import com.example.appfunctions.agent.domain.appfunction.ToolConverter +import javax.inject.Inject +import javax.inject.Singleton import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.JsonPrimitive import kotlinx.serialization.json.buildJsonArray import kotlinx.serialization.json.buildJsonObject -import javax.inject.Inject -import javax.inject.Singleton /** Implementation of [ToolConverter] for Gemini tools. */ @Singleton class GeminiToolConverter - @Inject - constructor() : ToolConverter { - override fun convert(tool: AppFunctionMetadata): JsonObject { - return convertToolToGeminiSchema(tool) - } +@Inject +constructor() : ToolConverter { + override fun convert(tool: AppFunctionMetadata): JsonObject { + return convertToolToGeminiSchema(tool) + } - override fun getToolName(tool: AppFunctionMetadata): String { - return getGeminiFunctionName(tool) - } + override fun getToolName(tool: AppFunctionMetadata): String { + return getGeminiFunctionName(tool) + } - private fun convertToolToGeminiSchema(tool: AppFunctionMetadata): JsonObject { - val combinedName = getGeminiFunctionName(tool) - return buildJsonObject { - put(KEY_NAME, JsonPrimitive(combinedName)) - put(KEY_DESCRIPTION, JsonPrimitive(tool.description)) - put( - KEY_PARAMETERS, - buildJsonObject { - put(KEY_TYPE, JsonPrimitive(VALUE_OBJECT)) - put( - KEY_PROPERTIES, - buildJsonObject { - tool.parameters.forEach { parameter -> - val typeSchema = - mapDataTypeToGeminiSchema( - parameter.dataType, - tool.components, - tool.id, - parameterName = parameter.name, + private fun convertToolToGeminiSchema(tool: AppFunctionMetadata): JsonObject { + val combinedName = getGeminiFunctionName(tool) + return buildJsonObject { + put(KEY_NAME, JsonPrimitive(combinedName)) + put(KEY_DESCRIPTION, JsonPrimitive(tool.description)) + put( + KEY_PARAMETERS, + buildJsonObject { + put(KEY_TYPE, JsonPrimitive(VALUE_OBJECT)) + put( + KEY_PROPERTIES, + buildJsonObject { + tool.parameters.forEach { parameter -> + val typeSchema = + mapDataTypeToGeminiSchema( + parameter.dataType, + tool.components, + tool.id, + parameterName = parameter.name + ) + put( + parameter.name, + buildJsonObject { + typeSchema.forEach { (key, value) -> put(key, value) } + put( + KEY_DESCRIPTION, + JsonPrimitive(parameter.description) ) - put( - parameter.name, - buildJsonObject { - typeSchema.forEach { (key, value) -> put(key, value) } - put(KEY_DESCRIPTION, JsonPrimitive(parameter.description)) - }, + } + ) + } + } + ) + val requiredParams = tool.parameters.filter { it.isRequired }.map { it.name } + if (requiredParams.isNotEmpty()) { + put( + KEY_REQUIRED, + buildJsonArray { + requiredParams.forEach { + add( + JsonPrimitive(it) ) } - }, + } ) - val requiredParams = tool.parameters.filter { it.isRequired }.map { it.name } - if (requiredParams.isNotEmpty()) { - put( - KEY_REQUIRED, - buildJsonArray { requiredParams.forEach { add(JsonPrimitive(it)) } }, - ) - } - }, - ) - } + } + } + ) } + } - private fun getGeminiFunctionName(tool: AppFunctionMetadata): String { - val baseName = tool.id.replace(INVALID_NAME_CHARS, TOOL_ID_SEPARATOR) - val components = baseName.split(TOOL_ID_SEPARATOR) - var result = "" + private fun getGeminiFunctionName(tool: AppFunctionMetadata): String { + val baseName = tool.id.replace(INVALID_NAME_CHARS, TOOL_ID_SEPARATOR) + val components = baseName.split(TOOL_ID_SEPARATOR) + var result = "" - // Build from right to left, keeping the most specific parts of the package/id - for (i in components.indices.reversed()) { - val component = components[i] - val separator = if (result.isEmpty()) "" else TOOL_ID_SEPARATOR + // Build from right to left, keeping the most specific parts of the package/id + for (i in components.indices.reversed()) { + val component = components[i] + val separator = if (result.isEmpty()) "" else TOOL_ID_SEPARATOR - val newLength = result.length + separator.length + component.length - if (newLength <= MAX_NAME_LENGTH) { - result = component + separator + result - } else { - break - } + val newLength = result.length + separator.length + component.length + if (newLength <= MAX_NAME_LENGTH) { + result = component + separator + result + } else { + break } - if (result.isEmpty()) { - result = baseName.takeLast(MAX_NAME_LENGTH) - } - return result } + if (result.isEmpty()) { + result = baseName.takeLast(MAX_NAME_LENGTH) + } + return result + } - private fun mapDataTypeToGeminiSchema( - dataType: AppFunctionDataTypeMetadata, - components: AppFunctionComponentsMetadata, - functionId: String, - visitedReferences: Set = emptySet(), - parameterName: String? = null, - ): JsonObject { - return when (dataType) { - is AppFunctionStringTypeMetadata -> - buildJsonObject { - put(KEY_TYPE, JsonPrimitive(VALUE_STRING)) - if (isFileReferenceParameter(parameterName)) { - put(KEY_FORMAT, JsonPrimitive(VALUE_FILE_REFERENCE)) - } - val enumValues = dataType.enumValues - if (!enumValues.isNullOrEmpty()) { - put( - KEY_ENUMS, - buildJsonArray { - for (enumValue in enumValues) { - add(JsonPrimitive(enumValue)) - } - }, - ) - } + private fun mapDataTypeToGeminiSchema( + dataType: AppFunctionDataTypeMetadata, + components: AppFunctionComponentsMetadata, + functionId: String, + visitedReferences: Set = emptySet(), + parameterName: String? = null + ): JsonObject { + return when (dataType) { + is AppFunctionStringTypeMetadata -> + buildJsonObject { + put(KEY_TYPE, JsonPrimitive(VALUE_STRING)) + if (isFileReferenceParameter(parameterName)) { + put(KEY_FORMAT, JsonPrimitive(VALUE_FILE_REFERENCE)) + } + val enumValues = dataType.enumValues + if (!enumValues.isNullOrEmpty()) { + put( + KEY_ENUMS, + buildJsonArray { + for (enumValue in enumValues) { + add(JsonPrimitive(enumValue)) + } + } + ) } - is AppFunctionLongTypeMetadata -> - buildJsonObject { put(KEY_TYPE, JsonPrimitive(VALUE_INTEGER)) } - is AppFunctionIntTypeMetadata -> + } + is AppFunctionLongTypeMetadata -> + buildJsonObject { put(KEY_TYPE, JsonPrimitive(VALUE_INTEGER)) } + is AppFunctionIntTypeMetadata -> + buildJsonObject { + put(KEY_TYPE, JsonPrimitive(VALUE_INTEGER)) + val enumValues = dataType.enumValues + if (!enumValues.isNullOrEmpty()) { + put( + KEY_ENUMS, + buildJsonArray { + for (enumValue in enumValues) { + add(JsonPrimitive(enumValue)) + } + } + ) + } + } + is AppFunctionBooleanTypeMetadata -> + buildJsonObject { put(KEY_TYPE, JsonPrimitive(VALUE_BOOLEAN)) } + is AppFunctionDoubleTypeMetadata -> + buildJsonObject { put(KEY_TYPE, JsonPrimitive(VALUE_NUMBER)) } + is AppFunctionFloatTypeMetadata -> + buildJsonObject { put(KEY_TYPE, JsonPrimitive(VALUE_NUMBER)) } + is AppFunctionArrayTypeMetadata -> + buildJsonObject { + put(KEY_TYPE, JsonPrimitive(VALUE_ARRAY)) + put( + KEY_ITEMS, + mapDataTypeToGeminiSchema( + dataType.itemType, + components, + functionId, + visitedReferences, + parameterName + ) + ) + } + is AppFunctionObjectTypeMetadata -> + if (dataType.qualifiedName == "android.net.Uri") { buildJsonObject { - put(KEY_TYPE, JsonPrimitive(VALUE_INTEGER)) - val enumValues = dataType.enumValues - if (!enumValues.isNullOrEmpty()) { - put( - KEY_ENUMS, - buildJsonArray { - for (enumValue in enumValues) { - add(JsonPrimitive(enumValue)) - } - }, - ) - } + put(KEY_TYPE, JsonPrimitive(VALUE_STRING)) + put(KEY_FORMAT, JsonPrimitive(VALUE_FILE_REFERENCE)) } - is AppFunctionBooleanTypeMetadata -> - buildJsonObject { put(KEY_TYPE, JsonPrimitive(VALUE_BOOLEAN)) } - is AppFunctionDoubleTypeMetadata -> - buildJsonObject { put(KEY_TYPE, JsonPrimitive(VALUE_NUMBER)) } - is AppFunctionFloatTypeMetadata -> - buildJsonObject { put(KEY_TYPE, JsonPrimitive(VALUE_NUMBER)) } - is AppFunctionArrayTypeMetadata -> + } else { buildJsonObject { - put(KEY_TYPE, JsonPrimitive(VALUE_ARRAY)) + put(KEY_TYPE, JsonPrimitive(VALUE_OBJECT)) put( - KEY_ITEMS, - mapDataTypeToGeminiSchema( - dataType.itemType, - components, - functionId, - visitedReferences, - parameterName, - ), + KEY_PROPERTIES, + buildJsonObject { + dataType.properties.forEach { (name, type) -> + put( + name, + mapDataTypeToGeminiSchema( + type, + components, + functionId, + visitedReferences, + parameterName = name + ) + ) + } + } ) - } - is AppFunctionObjectTypeMetadata -> - if (dataType.qualifiedName == "android.net.Uri") { - buildJsonObject { - put(KEY_TYPE, JsonPrimitive(VALUE_STRING)) - put(KEY_FORMAT, JsonPrimitive(VALUE_FILE_REFERENCE)) - } - } else { - buildJsonObject { - put(KEY_TYPE, JsonPrimitive(VALUE_OBJECT)) + if (dataType.required.isNotEmpty()) { put( - KEY_PROPERTIES, - buildJsonObject { - dataType.properties.forEach { (name, type) -> - put( - name, - mapDataTypeToGeminiSchema( - type, - components, - functionId, - visitedReferences, - parameterName = name, - ), + KEY_REQUIRED, + buildJsonArray { + dataType.required.forEach { name -> + add( + JsonPrimitive(name) ) } - }, + } ) - if (dataType.required.isNotEmpty()) { - put( - KEY_REQUIRED, - buildJsonArray { - dataType.required.forEach { name -> add(JsonPrimitive(name)) } - }, - ) - } } } - is AppFunctionReferenceTypeMetadata -> { - val referenceKey = dataType.referenceDataType - if (referenceKey == "android.net.Uri") { - return buildJsonObject { - put(KEY_TYPE, JsonPrimitive(VALUE_STRING)) - put(KEY_FORMAT, JsonPrimitive(VALUE_FILE_REFERENCE)) - } - } - if (visitedReferences.contains(referenceKey)) { - Log.d( - "GeminiToolConverter", - "Circular reference detected for $referenceKey in function $functionId. Breaking cycle.", - ) - return buildJsonObject { put(KEY_TYPE, JsonPrimitive(VALUE_OBJECT)) } + } + is AppFunctionReferenceTypeMetadata -> { + val referenceKey = dataType.referenceDataType + if (referenceKey == "android.net.Uri") { + return buildJsonObject { + put(KEY_TYPE, JsonPrimitive(VALUE_STRING)) + put(KEY_FORMAT, JsonPrimitive(VALUE_FILE_REFERENCE)) } - val objectType = - components.dataTypes[referenceKey] - ?: throw IllegalArgumentException( - "Reference type $referenceKey not found in components for function $functionId", - ) - mapDataTypeToGeminiSchema( - objectType, - components, - functionId, - visitedReferences + referenceKey, - parameterName, - ) } - else -> - throw IllegalArgumentException( - "Unsupported data type: $dataType for function $functionId", + if (visitedReferences.contains(referenceKey)) { + Log.d( + "GeminiToolConverter", + "Circular reference detected for $referenceKey in function $functionId. Breaking cycle." ) + return buildJsonObject { put(KEY_TYPE, JsonPrimitive(VALUE_OBJECT)) } + } + val objectType = + components.dataTypes[referenceKey] + ?: throw IllegalArgumentException( + "Reference type $referenceKey not found in components for function $functionId" + ) + mapDataTypeToGeminiSchema( + objectType, + components, + functionId, + visitedReferences + referenceKey, + parameterName + ) } + else -> + throw IllegalArgumentException( + "Unsupported data type: $dataType for function $functionId" + ) } + } - private fun isFileReferenceParameter(parameterName: String?): Boolean { - if (parameterName == null) return false - if (parameterName in KNOWN_FILE_REFERENCE_PARAM_NAMES) return true - return parameterName.endsWith("Uri", ignoreCase = true) || - parameterName.endsWith("Uris", ignoreCase = true) - } + private fun isFileReferenceParameter(parameterName: String?): Boolean { + if (parameterName == null) return false + if (parameterName in KNOWN_FILE_REFERENCE_PARAM_NAMES) return true + return parameterName.endsWith("Uri", ignoreCase = true) || + parameterName.endsWith("Uris", ignoreCase = true) + } - companion object { - private const val TOOL_ID_SEPARATOR = "_" - private const val KEY_NAME = "name" - private const val KEY_DESCRIPTION = "description" - private const val KEY_PARAMETERS = "parameters" - private const val KEY_TYPE = "type" - private const val KEY_FORMAT = "format" - private const val VALUE_FILE_REFERENCE = "file_reference" - private val KNOWN_FILE_REFERENCE_PARAM_NAMES = - setOf( - "wallpaperUri", - "imageUri", - "attachmentUri", - "ringtoneUri", - "profilePictureUri", - "audioUri", - ) - private const val VALUE_OBJECT = "object" - private const val KEY_PROPERTIES = "properties" - private const val KEY_REQUIRED = "required" - private const val VALUE_STRING = "string" - private const val VALUE_INTEGER = "integer" - private const val VALUE_BOOLEAN = "boolean" - private const val VALUE_NUMBER = "number" - private const val VALUE_ARRAY = "array" - private const val KEY_ITEMS = "items" - private const val KEY_ENUMS = "enums" - private val INVALID_NAME_CHARS = Regex("[^a-zA-Z0-9_]") - private const val MAX_NAME_LENGTH = 64 - } + companion object { + private const val TOOL_ID_SEPARATOR = "_" + private const val KEY_NAME = "name" + private const val KEY_DESCRIPTION = "description" + private const val KEY_PARAMETERS = "parameters" + private const val KEY_TYPE = "type" + private const val KEY_FORMAT = "format" + private const val VALUE_FILE_REFERENCE = "file_reference" + private val KNOWN_FILE_REFERENCE_PARAM_NAMES = + setOf( + "wallpaperUri", + "imageUri", + "attachmentUri", + "ringtoneUri", + "profilePictureUri", + "audioUri" + ) + private const val VALUE_OBJECT = "object" + private const val KEY_PROPERTIES = "properties" + private const val KEY_REQUIRED = "required" + private const val VALUE_STRING = "string" + private const val VALUE_INTEGER = "integer" + private const val VALUE_BOOLEAN = "boolean" + private const val VALUE_NUMBER = "number" + private const val VALUE_ARRAY = "array" + private const val KEY_ITEMS = "items" + private const val KEY_ENUMS = "enums" + private val INVALID_NAME_CHARS = Regex("[^a-zA-Z0-9_]") + private const val MAX_NAME_LENGTH = 64 } +} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt index 7e4b1c9..4d6ca94 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt @@ -20,7 +20,6 @@ import android.content.Context import android.content.Intent import android.net.Uri import android.util.Log -import androidx.appfunctions.AppFunctionException import androidx.appfunctions.metadata.AppFunctionArrayTypeMetadata import androidx.appfunctions.metadata.AppFunctionDataTypeMetadata import androidx.appfunctions.metadata.AppFunctionMetadata @@ -28,10 +27,6 @@ import androidx.appfunctions.metadata.AppFunctionObjectTypeMetadata import androidx.appfunctions.metadata.AppFunctionParameterMetadata import androidx.appfunctions.metadata.AppFunctionReferenceTypeMetadata import androidx.core.content.FileProvider -import java.io.File -import java.net.HttpURLConnection -import java.net.URL -import java.util.UUID import com.example.appfunctions.agent.data.LlmProviderName import com.example.appfunctions.agent.data.SettingsRepository import com.example.appfunctions.agent.data.db.entities.MessageAttachment @@ -53,6 +48,10 @@ import com.example.appfunctions.agent.domain.chat.UpdateThreadParams import com.example.appfunctions.agent.domain.chat.UpdateThreadUseCase import com.example.appfunctions.agent.domain.pendingintent.SavePendingIntentUseCase import dagger.hilt.android.qualifiers.ApplicationContext +import java.io.File +import java.net.HttpURLConnection +import java.net.URL +import java.util.UUID import javax.inject.Inject import javax.inject.Singleton import kotlinx.coroutines.CancellationException @@ -243,7 +242,7 @@ constructor( data class PendingIntentAction( val pendingIntentId: String, - val pendingIntent: PendingIntent, + val pendingIntent: PendingIntent ) : ExecuteToolCallsResult() object Error : ExecuteToolCallsResult() @@ -379,14 +378,14 @@ constructor( resolveRemoteFileReferencesRecursively( context = context, parametersMetadata = matchingTool.parameters, - inputs = rawConvertedInputs, + inputs = rawConvertedInputs ) } } catch (e: Exception) { completeMessageWithError( message.messageId, message.threadId, - "Failed to download remote file reference: ${e.message}", + "Failed to download remote file reference: ${e.message}" ) return ExecuteToolCallsResult.Error } @@ -450,16 +449,16 @@ constructor( functionId = toolCall.functionId, callId = toolCall.callId, result = - AppFunctionExceptionFormatter.format( - appFunctionException, - toolCall.functionId, - ), - ), + AppFunctionExceptionFormatter.format( + appFunctionException, + toolCall.functionId + ) + ) ) } else { throw IllegalStateException( "Tool execution failed for ${toolCall.functionId}: ${exception.message}", - exception, + exception ) } } @@ -488,7 +487,7 @@ constructor( private fun resolveRemoteFileReferencesRecursively( context: Context, parametersMetadata: List, - inputs: Map, + inputs: Map ): Map { val paramMap = parametersMetadata.associateBy { it.name } return inputs.mapValues { (key, value) -> @@ -501,13 +500,18 @@ constructor( context: Context, dataType: AppFunctionDataTypeMetadata?, value: Any, - paramName: String?, + paramName: String? ): Any { return when (value) { is String -> { val shouldResolve = isFileReferenceParameter(paramName) || isUriMetadata(dataType) - if (shouldResolve && (value.startsWith("http://") || value.startsWith("https://"))) { + if (shouldResolve && ( + value.startsWith( + "http://" + ) || value.startsWith("https://") + ) + ) { downloadRemoteFileToContentUri(context, value) } else { value @@ -518,7 +522,16 @@ constructor( val k = entry.key as String val propType = (dataType as? AppFunctionObjectTypeMetadata)?.properties?.get(k) - k to (entry.value?.let { resolveValueRecursively(context, propType, it, k) } ?: "") + k to ( + entry.value?.let { + resolveValueRecursively( + context, + propType, + it, + k + ) + } ?: "" + ) } } is List<*> -> { @@ -559,7 +572,7 @@ constructor( FileProvider.getUriForFile( context, "${context.packageName}.fileprovider", - file, + file ) return contentUri.toString() } finally { @@ -570,7 +583,7 @@ constructor( private fun grantContentUriPermissionsRecursively( context: Context, targetPackageName: String, - value: Any?, + value: Any? ) { when (value) { is String -> { @@ -580,13 +593,17 @@ constructor( context.grantUriPermission( targetPackageName, uri, - Intent.FLAG_GRANT_READ_URI_PERMISSION, + Intent.FLAG_GRANT_READ_URI_PERMISSION ) } } } - is Map<*, *> -> value.values.forEach { grantContentUriPermissionsRecursively(context, targetPackageName, it) } - is List<*> -> value.forEach { grantContentUriPermissionsRecursively(context, targetPackageName, it) } + is Map<*, *> -> value.values.forEach { + grantContentUriPermissionsRecursively(context, targetPackageName, it) + } + is List<*> -> value.forEach { + grantContentUriPermissionsRecursively(context, targetPackageName, it) + } } } @@ -612,7 +629,7 @@ constructor( "attachmentUri", "ringtoneUri", "profilePictureUri", - "audioUri", + "audioUri" ) } } From c23690743d22589d9878c4ef2cf4efbf932cd8d9 Mon Sep 17 00:00:00 2001 From: Caleb Areeveso Date: Fri, 10 Jul 2026 11:39:11 +0000 Subject: [PATCH 05/16] fix: address gemini-code-assist bot code review feedback Change-Id: I7d56a269f7965e54ce98f84595a12f3035891056 --- .../AppFunctionInstrumentationTest.kt | 1 - ChatApp/shared/build.gradle.kts | 1 - .../example/chatapp/BaseChatApplication.kt | 1 + .../BaseChatAppFunctionService.kt | 18 +++-- .../com/example/chatapp/appfunctions/Util.kt | 4 +- .../chatapp/data/WallpaperRepository.kt | 76 ++++++++++--------- .../com/example/chatapp/util/Linkify.kt | 31 ++++---- .../agent/domain/AgentOrchestrator.kt | 6 +- 8 files changed, 78 insertions(+), 60 deletions(-) diff --git a/ChatApp/app/src/androidTest/kotlin/com/example/chatapp/appfunctions/AppFunctionInstrumentationTest.kt b/ChatApp/app/src/androidTest/kotlin/com/example/chatapp/appfunctions/AppFunctionInstrumentationTest.kt index 74e7e8e..4678127 100644 --- a/ChatApp/app/src/androidTest/kotlin/com/example/chatapp/appfunctions/AppFunctionInstrumentationTest.kt +++ b/ChatApp/app/src/androidTest/kotlin/com/example/chatapp/appfunctions/AppFunctionInstrumentationTest.kt @@ -24,7 +24,6 @@ import androidx.appfunctions.ExecuteAppFunctionRequest import androidx.appfunctions.ExecuteAppFunctionResponse import androidx.test.core.app.ApplicationProvider import androidx.test.ext.junit.runners.AndroidJUnit4 -import com.example.chatapp.appfunctions.ChatAppFunctionService import com.example.chatapp.data.MessageRepository import com.example.chatapp.data.RecipientsRepository import com.google.common.truth.Truth.assertThat diff --git a/ChatApp/shared/build.gradle.kts b/ChatApp/shared/build.gradle.kts index b9aba68..5a720e6 100644 --- a/ChatApp/shared/build.gradle.kts +++ b/ChatApp/shared/build.gradle.kts @@ -48,5 +48,4 @@ dependencies { // App functions implementation(libs.androidx.appfunctions) ksp(libs.androidx.appfunctions.compiler) - } diff --git a/ChatApp/shared/src/main/kotlin/com/example/chatapp/BaseChatApplication.kt b/ChatApp/shared/src/main/kotlin/com/example/chatapp/BaseChatApplication.kt index b051a26..4d53914 100644 --- a/ChatApp/shared/src/main/kotlin/com/example/chatapp/BaseChatApplication.kt +++ b/ChatApp/shared/src/main/kotlin/com/example/chatapp/BaseChatApplication.kt @@ -16,4 +16,5 @@ package com.example.chatapp import android.app.Application + abstract class BaseChatApplication : Application() diff --git a/ChatApp/shared/src/main/kotlin/com/example/chatapp/appfunctions/BaseChatAppFunctionService.kt b/ChatApp/shared/src/main/kotlin/com/example/chatapp/appfunctions/BaseChatAppFunctionService.kt index 7e565b1..cc13cfd 100644 --- a/ChatApp/shared/src/main/kotlin/com/example/chatapp/appfunctions/BaseChatAppFunctionService.kt +++ b/ChatApp/shared/src/main/kotlin/com/example/chatapp/appfunctions/BaseChatAppFunctionService.kt @@ -19,20 +19,20 @@ import android.app.PendingIntent import android.content.Intent import android.net.Uri import androidx.annotation.RequiresApi +import androidx.appfunctions.AppFunction import androidx.appfunctions.AppFunctionAppUnknownException import androidx.appfunctions.AppFunctionElementNotFoundException import androidx.appfunctions.AppFunctionInvalidArgumentException import androidx.appfunctions.AppFunctionService import androidx.appfunctions.AppFunctionServiceEntryPoint import androidx.appfunctions.AppFunctionStringValueConstraint -import androidx.appfunctions.AppFunction import com.example.chatapp.data.CallManager import com.example.chatapp.data.MessageRepository import com.example.chatapp.data.RecipientsRepository import com.example.chatapp.data.WallpaperRepository import dagger.hilt.android.AndroidEntryPoint -import javax.inject.Inject import kotlinx.coroutines.CancellationException +import javax.inject.Inject /** * Service entry point for chat-related AppFunctions such as searching contacts, sending messages, and making calls. @@ -45,8 +45,11 @@ import kotlinx.coroutines.CancellationException ) abstract class BaseChatAppFunctionService : AppFunctionService() { @Inject lateinit var messageRepository: MessageRepository + @Inject lateinit var recipientsRepository: RecipientsRepository + @Inject lateinit var callManager: CallManager + @Inject lateinit var wallpaperRepository: WallpaperRepository /** @@ -155,9 +158,7 @@ abstract class BaseChatAppFunctionService : AppFunctionService() { * @throws AppFunctionElementNotFoundException If no recipient exists for endpointValue. If thrown, call "searchContacts" to find the correct ID. */ @AppFunction(isDescribedByKDoc = true) - suspend fun makeCall( - endpointValue: String, - ): PendingIntent { + suspend fun makeCall(endpointValue: String): PendingIntent { val recipient = recipientsRepository.getRecipientById(endpointValue) ?: throw AppFunctionElementNotFoundException( @@ -192,8 +193,11 @@ abstract class BaseChatAppFunctionService : AppFunctionService() { ?: recipientsRepository.searchAny(chatId, maxCount = 1).firstOrNull()?.endpointValue ?: chatId val inputStream = - contentResolver.openInputStream(wallpaperUri) - ?: throw AppFunctionInvalidArgumentException("Cannot open wallpaper stream") + try { + contentResolver.openInputStream(wallpaperUri) + } catch (e: Exception) { + throw AppFunctionInvalidArgumentException("Cannot open wallpaper stream: ${e.message}") + } ?: throw AppFunctionInvalidArgumentException("Cannot open wallpaper stream") return inputStream.use { stream -> wallpaperRepository.setWallpaper(resolvedId, stream) } diff --git a/ChatApp/shared/src/main/kotlin/com/example/chatapp/appfunctions/Util.kt b/ChatApp/shared/src/main/kotlin/com/example/chatapp/appfunctions/Util.kt index 81d0354..51ccd17 100644 --- a/ChatApp/shared/src/main/kotlin/com/example/chatapp/appfunctions/Util.kt +++ b/ChatApp/shared/src/main/kotlin/com/example/chatapp/appfunctions/Util.kt @@ -13,12 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package com.example.chatapp.appfunctions import androidx.appfunctions.AppFunctionSerializable - /** * Represents a result from a contact or group search. */ @@ -67,4 +65,4 @@ data class ChatGroup( val name: String, /** List of members belonging to the group. */ val recipients: List, -) \ No newline at end of file +) diff --git a/ChatApp/shared/src/main/kotlin/com/example/chatapp/data/WallpaperRepository.kt b/ChatApp/shared/src/main/kotlin/com/example/chatapp/data/WallpaperRepository.kt index 0122f2f..e8a85f6 100644 --- a/ChatApp/shared/src/main/kotlin/com/example/chatapp/data/WallpaperRepository.kt +++ b/ChatApp/shared/src/main/kotlin/com/example/chatapp/data/WallpaperRepository.kt @@ -21,57 +21,65 @@ import dagger.Module import dagger.hilt.InstallIn import dagger.hilt.android.qualifiers.ApplicationContext import dagger.hilt.components.SingletonComponent -import java.io.File -import java.io.InputStream -import javax.inject.Inject -import javax.inject.Singleton import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.flowOn import kotlinx.coroutines.flow.map import kotlinx.coroutines.flow.update import kotlinx.coroutines.withContext +import java.io.File +import java.io.InputStream +import javax.inject.Inject +import javax.inject.Singleton interface WallpaperRepository { fun getWallpaper(chatId: String): Flow - suspend fun setWallpaper(chatId: String, inputStream: InputStream): Boolean + + suspend fun setWallpaper( + chatId: String, + inputStream: InputStream, + ): Boolean } @Singleton -class WallpaperRepositoryImpl @Inject constructor( - @ApplicationContext private val context: Context, -) : WallpaperRepository { - private val wallpapers = MutableStateFlow>(emptyMap()) +class WallpaperRepositoryImpl + @Inject + constructor( + @ApplicationContext private val context: Context, + ) : WallpaperRepository { + private val wallpapers = MutableStateFlow>(emptyMap()) - override fun getWallpaper(chatId: String): Flow { - return wallpapers.map { map -> - map[chatId] ?: run { - val dir = File(context.filesDir, "wallpapers") - dir.listFiles { f -> f.name.startsWith("wallpaper_${chatId}_") } - ?.maxByOrNull { it.lastModified() } - ?.absolutePath - } + override fun getWallpaper(chatId: String): Flow { + return wallpapers.map { map -> + map[chatId] ?: run { + val dir = File(context.filesDir, "wallpapers") + dir.listFiles { f -> f.name.startsWith("wallpaper_${chatId}_") } + ?.maxByOrNull { it.lastModified() } + ?.absolutePath + } + }.flowOn(Dispatchers.IO) } - } - override suspend fun setWallpaper( - chatId: String, - inputStream: InputStream, - ): Boolean = withContext(Dispatchers.IO) { - try { - val dir = File(context.filesDir, "wallpapers").apply { mkdirs() } - dir.listFiles { f -> f.name.startsWith("wallpaper_${chatId}_") }?.forEach { it.delete() } - val file = File(dir, "wallpaper_${chatId}_${System.currentTimeMillis()}.jpg") - file.outputStream().use { output -> - inputStream.copyTo(output) + override suspend fun setWallpaper( + chatId: String, + inputStream: InputStream, + ): Boolean = + withContext(Dispatchers.IO) { + try { + val dir = File(context.filesDir, "wallpapers").apply { mkdirs() } + dir.listFiles { f -> f.name.startsWith("wallpaper_${chatId}_") }?.forEach { it.delete() } + val file = File(dir, "wallpaper_${chatId}_${System.currentTimeMillis()}.jpg") + file.outputStream().use { output -> + inputStream.copyTo(output) + } + wallpapers.update { current -> current + (chatId to file.absolutePath) } + true + } catch (e: Exception) { + false + } } - wallpapers.update { current -> current + (chatId to file.absolutePath) } - true - } catch (e: Exception) { - false - } } -} @Module @InstallIn(SingletonComponent::class) diff --git a/ChatApp/shared/src/main/kotlin/com/example/chatapp/util/Linkify.kt b/ChatApp/shared/src/main/kotlin/com/example/chatapp/util/Linkify.kt index f0a8f93..453ff4f 100644 --- a/ChatApp/shared/src/main/kotlin/com/example/chatapp/util/Linkify.kt +++ b/ChatApp/shared/src/main/kotlin/com/example/chatapp/util/Linkify.kt @@ -24,7 +24,10 @@ import androidx.compose.ui.text.TextLinkStyles import androidx.compose.ui.text.buildAnnotatedString import androidx.compose.ui.text.style.TextDecoration -fun linkifyString(text: String, linkColor: Color? = null): AnnotatedString { +fun linkifyString( + text: String, + linkColor: Color? = null, +): AnnotatedString { val matcher = Patterns.WEB_URL.matcher(text) var lastIndex = 0 return buildAnnotatedString { @@ -35,22 +38,24 @@ fun linkifyString(text: String, linkColor: Color? = null): AnnotatedString { append(text.substring(lastIndex, start)) - val uriStr = if (url.startsWith("http://", ignoreCase = true) || url.startsWith("https://", ignoreCase = true)) { - url - } else { - "https://$url" - } - - val linkStyle = SpanStyle( - color = linkColor ?: Color.Unspecified, - textDecoration = TextDecoration.Underline - ) + val uriStr = + if (url.startsWith("http://", ignoreCase = true) || url.startsWith("https://", ignoreCase = true)) { + url + } else { + "https://$url" + } + + val linkStyle = + SpanStyle( + color = linkColor ?: Color.Unspecified, + textDecoration = TextDecoration.Underline, + ) pushLink( LinkAnnotation.Url( url = uriStr, - styles = TextLinkStyles(style = linkStyle) - ) + styles = TextLinkStyles(style = linkStyle), + ), ) append(url) pop() diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt index 4d6ca94..8fd76ef 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt @@ -546,7 +546,11 @@ constructor( private fun downloadRemoteFileToContentUri(context: Context, urlString: String): String { val url = URL(urlString) - val connection = url.openConnection() as HttpURLConnection + val connection = + (url.openConnection() as HttpURLConnection).apply { + connectTimeout = 10000 + readTimeout = 15000 + } try { connection.connect() val contentType = connection.contentType ?: "" From 7c2f3d7c7dc1676e43bfd38936e2e0cd831f000e Mon Sep 17 00:00:00 2001 From: Caleb Areeveso Date: Tue, 14 Jul 2026 11:36:51 +0000 Subject: [PATCH 06/16] style: apply spotless formatting and rename service files without spotless suppressions Change-Id: Ibee730a08f815dff8cc600845af1d2da3636e78b --- agent/app/build.gradle.kts | 9 +- ...ns.kt => BaseBuiltInAppFunctionService.kt} | 276 +++-- ...tions.kt => BaseFakeAppFunctionService.kt} | 4 +- .../agent/data/GeminiProviderImpl.kt | 354 +++---- .../agent/data/GeminiToolConverter.kt | 458 ++++----- .../agent/data/db/entities/MessageEntity.kt | 38 +- .../appfunctions/agent/di/DataModule.kt | 20 +- .../agent/domain/AgentOrchestrator.kt | 951 +++++++++--------- .../AppFunctionExceptionFormatter.kt | 5 +- .../agent/domain/chat/SendMessageUseCase.kt | 68 +- .../ui/screens/agentdemo/AgentDemoScreen.kt | 340 ++++--- .../debugging/FunctionsFoundContent.kt | 1 - .../MessageAttachmentConverterTest.kt | 7 +- .../agent/domain/AgentOrchestratorTest.kt | 321 +++--- .../AppFunctionExceptionFormatterTest.kt | 3 +- .../ExecuteAppFunctionUseCaseTest.kt | 51 +- 16 files changed, 1467 insertions(+), 1439 deletions(-) rename agent/app/src/main/java/com/example/appfunctions/agent/data/{BuiltInAppFunctions.kt => BaseBuiltInAppFunctionService.kt} (56%) rename agent/app/src/main/java/com/example/appfunctions/agent/data/{FakeAppFunctions.kt => BaseFakeAppFunctionService.kt} (96%) diff --git a/agent/app/build.gradle.kts b/agent/app/build.gradle.kts index 75f0cac..ec6f854 100644 --- a/agent/app/build.gradle.kts +++ b/agent/app/build.gradle.kts @@ -61,13 +61,14 @@ android { dimension = "mode" buildConfigField("Boolean", "IS_RETAIL", "true") - val containsRetail = gradle.startParameter.taskNames.any { - it.contains("Retail", ignoreCase = true) - } + val containsRetail = + gradle.startParameter.taskNames.any { + it.contains("Retail", ignoreCase = true) + } val apiKey = project.findProperty("GEMINI_API_KEY") as? String ?: "" if (containsRetail && apiKey.isEmpty()) { throw GradleException( - "GEMINI_API_KEY project property is required for retail builds. Pass it using -PGEMINI_API_KEY=your_key" + "GEMINI_API_KEY project property is required for retail builds. Pass it using -PGEMINI_API_KEY=your_key", ) } buildConfigField("String", "GEMINI_API_KEY", "\"$apiKey\"") diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/BuiltInAppFunctions.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/BaseBuiltInAppFunctionService.kt similarity index 56% rename from agent/app/src/main/java/com/example/appfunctions/agent/data/BuiltInAppFunctions.kt rename to agent/app/src/main/java/com/example/appfunctions/agent/data/BaseBuiltInAppFunctionService.kt index 0d0f61d..912426e 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/BuiltInAppFunctions.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/BaseBuiltInAppFunctionService.kt @@ -25,7 +25,6 @@ import android.location.LocationManager import android.util.Base64 import androidx.annotation.RequiresApi import androidx.appfunctions.AppFunction -import androidx.appfunctions.AppFunctionContext import androidx.appfunctions.AppFunctionSerializable import androidx.appfunctions.AppFunctionService import androidx.appfunctions.AppFunctionServiceEntryPoint @@ -35,17 +34,17 @@ import androidx.datastore.preferences.core.stringPreferencesKey import com.example.appfunctions.agent.BuildConfig import com.example.appfunctions.agent.di.settingsDataStore import dagger.hilt.android.AndroidEntryPoint +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.withContext +import org.json.JSONArray +import org.json.JSONObject import java.io.File import java.net.HttpURLConnection import java.net.URL import java.util.UUID import kotlin.coroutines.resume import kotlin.coroutines.suspendCoroutine -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.flow.first -import kotlinx.coroutines.withContext -import org.json.JSONArray -import org.json.JSONObject /** Built-in AppFunctions for location and geocoding services. */ @RequiresApi(36) @@ -63,9 +62,7 @@ abstract class BaseBuiltInAppFunctionService : AppFunctionService() { * @return The latitude and longitude coordinates of the address, or null if geocoding fails. */ @AppFunction(isDescribedByKDoc = true) - suspend fun geocodeAddress( - address: String, - ): LatLng? { + suspend fun geocodeAddress(address: String): LatLng? { if (!Geocoder.isPresent()) { return null } @@ -83,7 +80,7 @@ abstract class BaseBuiltInAppFunctionService : AppFunctionService() { val location = addresses.firstOrNull() if (location != null) { continuation.resume( - LatLng(location.latitude, location.longitude) + LatLng(location.latitude, location.longitude), ) } else { continuation.resume(null) @@ -93,7 +90,7 @@ abstract class BaseBuiltInAppFunctionService : AppFunctionService() { override fun onError(errorMessage: String?) { continuation.resume(null) } - } + }, ) } } catch (e: Exception) { @@ -165,7 +162,7 @@ abstract class BaseBuiltInAppFunctionService : AppFunctionService() { /** The latitude coordinate. */ val latitude: Double, /** The longitude coordinate. */ - val longitude: Double + val longitude: Double, ) /** @@ -178,148 +175,149 @@ abstract class BaseBuiltInAppFunctionService : AppFunctionService() { @AppFunction(isDescribedByKDoc = true) suspend fun generateImage( prompt: String, - aspectRatio: String? = null - ): GeneratedImageResult = withContext(Dispatchers.IO) { - val context = this@BaseBuiltInAppFunctionService - val apiKey = - context.settingsDataStore.data - .first()[stringPreferencesKey("gemini_api_key")] - ?.takeIf { it.isNotBlank() } - ?: BuildConfig.GEMINI_API_KEY.takeIf { it.isNotBlank() } - if (apiKey.isNullOrBlank()) { - throw IllegalStateException( - "Gemini API key is not configured. Please set gemini_api_key in settings." - ) - } - - val requestJson = - JSONObject().apply { - put( - "contents", - JSONArray().apply { - put( - JSONObject().apply { - put( - "parts", - JSONArray().apply { - put(JSONObject().apply { put("text", prompt) }) - } - ) - } - ) - } + aspectRatio: String? = null, + ): GeneratedImageResult = + withContext(Dispatchers.IO) { + val context = this@BaseBuiltInAppFunctionService + val apiKey = + context.settingsDataStore.data + .first()[stringPreferencesKey("gemini_api_key")] + ?.takeIf { it.isNotBlank() } + ?: BuildConfig.GEMINI_API_KEY.takeIf { it.isNotBlank() } + if (apiKey.isNullOrBlank()) { + throw IllegalStateException( + "Gemini API key is not configured. Please set gemini_api_key in settings.", ) - put( - "generationConfig", - JSONObject().apply { - put("responseModalities", JSONArray().apply { put("IMAGE") }) - if (!aspectRatio.isNullOrBlank()) { + } + + val requestJson = + JSONObject().apply { + put( + "contents", + JSONArray().apply { put( - "imageConfig", JSONObject().apply { - put("aspectRatio", aspectRatio) - } + put( + "parts", + JSONArray().apply { + put(JSONObject().apply { put("text", prompt) }) + }, + ) + }, ) - } - } - ) - } + }, + ) + put( + "generationConfig", + JSONObject().apply { + put("responseModalities", JSONArray().apply { put("IMAGE") }) + if (!aspectRatio.isNullOrBlank()) { + put( + "imageConfig", + JSONObject().apply { + put("aspectRatio", aspectRatio) + }, + ) + } + }, + ) + } - val endpointUrl = - "https://generativelanguage.googleapis.com/v1beta/models/gemini-3.1-flash-image-preview:generateContent?key=$apiKey" - val url = URL(endpointUrl) - val connection = - (url.openConnection() as HttpURLConnection).apply { - requestMethod = "POST" - setRequestProperty("Content-Type", "application/json") - doOutput = true - connectTimeout = 30000 - readTimeout = 60000 - } + val endpointUrl = + "https://generativelanguage.googleapis.com/v1beta/models/gemini-3.1-flash-image-preview:generateContent?key=$apiKey" + val url = URL(endpointUrl) + val connection = + (url.openConnection() as HttpURLConnection).apply { + requestMethod = "POST" + setRequestProperty("Content-Type", "application/json") + doOutput = true + connectTimeout = 30000 + readTimeout = 60000 + } - try { - connection.outputStream.use { os -> - os.write(requestJson.toString().toByteArray(Charsets.UTF_8)) - } + try { + connection.outputStream.use { os -> + os.write(requestJson.toString().toByteArray(Charsets.UTF_8)) + } - val responseCode = connection.responseCode - if (responseCode != HttpURLConnection.HTTP_OK) { - val errorBody = - connection.errorStream?.bufferedReader()?.use { it.readText() } - ?: "HTTP $responseCode" - throw IllegalStateException( - "Image generation failed ($responseCode): $errorBody" - ) - } + val responseCode = connection.responseCode + if (responseCode != HttpURLConnection.HTTP_OK) { + val errorBody = + connection.errorStream?.bufferedReader()?.use { it.readText() } + ?: "HTTP $responseCode" + throw IllegalStateException( + "Image generation failed ($responseCode): $errorBody", + ) + } - val responseText = connection.inputStream.bufferedReader().use { it.readText() } - val responseJson = JSONObject(responseText) - val candidates = responseJson.optJSONArray("candidates") - if (candidates == null || candidates.length() == 0) { - throw IllegalStateException( - "No candidates returned from Gemini image generation API" - ) - } + val responseText = connection.inputStream.bufferedReader().use { it.readText() } + val responseJson = JSONObject(responseText) + val candidates = responseJson.optJSONArray("candidates") + if (candidates == null || candidates.length() == 0) { + throw IllegalStateException( + "No candidates returned from Gemini image generation API", + ) + } - val parts = - candidates - .getJSONObject(0) - .optJSONObject("content") - ?.optJSONArray("parts") - if (parts == null || parts.length() == 0) { - throw IllegalStateException( - "No parts returned in candidate content. Gemini response: $responseText" - ) - } + val parts = + candidates + .getJSONObject(0) + .optJSONObject("content") + ?.optJSONArray("parts") + if (parts == null || parts.length() == 0) { + throw IllegalStateException( + "No parts returned in candidate content. Gemini response: $responseText", + ) + } - var base64Data: String? = null - var mimeType = "image/png" - for (i in 0 until parts.length()) { - val part = parts.getJSONObject(i) - val inlineData = - part.optJSONObject("inlineData") - ?: part.optJSONObject("inline_data") - if (inlineData != null) { - base64Data = inlineData.optString("data") - val returnedMime = - inlineData - .optString("mimeType") - .takeIf { it.isNotBlank() } - ?: inlineData.optString("mime_type") - if (returnedMime.isNotBlank()) { - mimeType = returnedMime + var base64Data: String? = null + var mimeType = "image/png" + for (i in 0 until parts.length()) { + val part = parts.getJSONObject(i) + val inlineData = + part.optJSONObject("inlineData") + ?: part.optJSONObject("inline_data") + if (inlineData != null) { + base64Data = inlineData.optString("data") + val returnedMime = + inlineData + .optString("mimeType") + .takeIf { it.isNotBlank() } + ?: inlineData.optString("mime_type") + if (returnedMime.isNotBlank()) { + mimeType = returnedMime + } + break } - break } - } - if (base64Data.isNullOrBlank()) { - throw IllegalStateException( - "No inlineData image found in response parts. Gemini response: $responseText" - ) - } + if (base64Data.isNullOrBlank()) { + throw IllegalStateException( + "No inlineData image found in response parts. Gemini response: $responseText", + ) + } - val imageBytes = Base64.decode(base64Data, Base64.DEFAULT) - val extension = - if (mimeType.contains("jpeg") || mimeType.contains("jpg")) "jpg" else "png" - val cachedFile = - File( - context.cacheDir, - "generated_${UUID.randomUUID()}.$extension" - ) - cachedFile.writeBytes(imageBytes) + val imageBytes = Base64.decode(base64Data, Base64.DEFAULT) + val extension = + if (mimeType.contains("jpeg") || mimeType.contains("jpg")) "jpg" else "png" + val cachedFile = + File( + context.cacheDir, + "generated_${UUID.randomUUID()}.$extension", + ) + cachedFile.writeBytes(imageBytes) - val authority = "${context.packageName}.fileprovider" - val contentUri = FileProvider.getUriForFile(context, authority, cachedFile) - GeneratedImageResult( - imageUri = contentUri.toString(), - mimeType = mimeType, - prompt = prompt - ) - } finally { - connection.disconnect() + val authority = "${context.packageName}.fileprovider" + val contentUri = FileProvider.getUriForFile(context, authority, cachedFile) + GeneratedImageResult( + imageUri = contentUri.toString(), + mimeType = mimeType, + prompt = prompt, + ) + } finally { + connection.disconnect() + } } - } /** Represents the result of an image generation request. */ @AppFunctionSerializable @@ -329,6 +327,6 @@ abstract class BaseBuiltInAppFunctionService : AppFunctionService() { /** The MIME type of the generated image. */ val mimeType: String, /** The original prompt used to generate the image. */ - val prompt: String + val prompt: String, ) } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/FakeAppFunctions.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/BaseFakeAppFunctionService.kt similarity index 96% rename from agent/app/src/main/java/com/example/appfunctions/agent/data/FakeAppFunctions.kt rename to agent/app/src/main/java/com/example/appfunctions/agent/data/BaseFakeAppFunctionService.kt index 1f82650..2048ee9 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/FakeAppFunctions.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/BaseFakeAppFunctionService.kt @@ -37,9 +37,7 @@ abstract class BaseFakeAppFunctionService : AppFunctionService() { * @return The test response containing output data. */ @AppFunction(isDescribedByKDoc = true) - suspend fun fakeFunction( - params: FakeParams, - ): FakeResponse { + suspend fun fakeFunction(params: FakeParams): FakeResponse { return FakeResponse("success") } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt index 177c237..4627e76 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt @@ -29,9 +29,6 @@ import io.ktor.client.statement.HttpResponse import io.ktor.client.statement.bodyAsText import io.ktor.http.ContentType import io.ktor.http.contentType -import java.time.LocalDate -import javax.inject.Inject -import javax.inject.Singleton import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonArray import kotlinx.serialization.json.JsonElement @@ -42,222 +39,225 @@ import kotlinx.serialization.json.jsonArray import kotlinx.serialization.json.jsonObject import kotlinx.serialization.json.jsonPrimitive import kotlinx.serialization.json.put +import java.time.LocalDate +import javax.inject.Inject +import javax.inject.Singleton @Singleton class GeminiProviderImpl -@Inject -constructor( - private val httpClient: HttpClient, - private val toolConverter: GeminiToolConverter -) : LlmProvider { - override suspend fun generateResponse( - previousInteractionId: String?, - input: LlmInput, - tools: List, - apiKey: String, - modelName: String - ): LlmResponse { - val sortedTools = tools.sortedByDescending { it.id.startsWith(it.packageName) } - val uniqueTools = sortedTools.distinctBy { toolConverter.getToolName(it) } - val convertedTools = - uniqueTools.mapNotNull { tool -> - try { - buildJsonObject { - put(KEY_TYPE, JsonPrimitive(VALUE_FUNCTION)) - val functionSchema = toolConverter.convert(tool) - functionSchema.forEach { (key, value) -> put(key, value) } + @Inject + constructor( + private val httpClient: HttpClient, + private val toolConverter: GeminiToolConverter, + ) : LlmProvider { + override suspend fun generateResponse( + previousInteractionId: String?, + input: LlmInput, + tools: List, + apiKey: String, + modelName: String, + ): LlmResponse { + val sortedTools = tools.sortedByDescending { it.id.startsWith(it.packageName) } + val uniqueTools = sortedTools.distinctBy { toolConverter.getToolName(it) } + val convertedTools = + uniqueTools.mapNotNull { tool -> + try { + buildJsonObject { + put(KEY_TYPE, JsonPrimitive(VALUE_FUNCTION)) + val functionSchema = toolConverter.convert(tool) + functionSchema.forEach { (key, value) -> put(key, value) } + } + } catch (e: IllegalArgumentException) { + Log.e(TAG, "Failed to convert tool ${tool.id}: ${e.message}", e) + null } - } catch (e: IllegalArgumentException) { - Log.e(TAG, "Failed to convert tool ${tool.id}: ${e.message}", e) - null } - } - - val requestBody = - buildJsonObject { - put(KEY_MODEL, JsonPrimitive(modelName)) - put(KEY_SYSTEM_INSTRUCTION, JsonPrimitive(getSystemInstruction())) + val requestBody = + buildJsonObject { + put(KEY_MODEL, JsonPrimitive(modelName)) - when (input) { - is LlmInput.ToolResponse -> { - val inputElement = - kotlinx.serialization.json.buildJsonArray { - input.outputs.forEach { output -> - val matchingTool = tools.find { it.id == output.functionId } - val mappedName = - if (matchingTool != null) { - toolConverter.getToolName(matchingTool) - } else { - output.functionId - } + put(KEY_SYSTEM_INSTRUCTION, JsonPrimitive(getSystemInstruction())) - add( - buildJsonObject { - put("type", "function_result") - put("name", mappedName) - if (output.callId.isNotEmpty()) { - put("call_id", output.callId) + when (input) { + is LlmInput.ToolResponse -> { + val inputElement = + kotlinx.serialization.json.buildJsonArray { + input.outputs.forEach { output -> + val matchingTool = tools.find { it.id == output.functionId } + val mappedName = + if (matchingTool != null) { + toolConverter.getToolName(matchingTool) + } else { + output.functionId } - put("result", output.result) - } - ) + + add( + buildJsonObject { + put("type", "function_result") + put("name", mappedName) + if (output.callId.isNotEmpty()) { + put("call_id", output.callId) + } + put("result", output.result) + }, + ) + } } - } - put(KEY_INPUT, inputElement) + put(KEY_INPUT, inputElement) + } + is LlmInput.UserMessage -> { + put(KEY_INPUT, JsonPrimitive(input.text)) + } } - is LlmInput.UserMessage -> { - put(KEY_INPUT, JsonPrimitive(input.text)) + + if (convertedTools.isNotEmpty()) { + put(KEY_TOOLS, kotlinx.serialization.json.JsonArray(convertedTools)) } - } - if (convertedTools.isNotEmpty()) { - put(KEY_TOOLS, kotlinx.serialization.json.JsonArray(convertedTools)) + if (previousInteractionId != null) { + put(KEY_PREVIOUS_INTERACTION_ID, JsonPrimitive(previousInteractionId)) + } } - if (previousInteractionId != null) { - put(KEY_PREVIOUS_INTERACTION_ID, JsonPrimitive(previousInteractionId)) - } - } + Log.d(TAG, "Gemini Request Body: $requestBody") - Log.d(TAG, "Gemini Request Body: $requestBody") - - val response: HttpResponse = - try { - httpClient.post(GEMINI_INTERACTIONS_URL) { - contentType(ContentType.Application.Json) - header(HEADER_API_KEY, apiKey) - header(HEADER_API_REVISION, VALUE_API_REVISION) - setBody(requestBody) + val response: HttpResponse = + try { + httpClient.post(GEMINI_INTERACTIONS_URL) { + contentType(ContentType.Application.Json) + header(HEADER_API_KEY, apiKey) + header(HEADER_API_REVISION, VALUE_API_REVISION) + setBody(requestBody) + } + } catch (e: Exception) { + return LlmResponse.Error("Network error: ${e.message}") } - } catch (e: Exception) { - return LlmResponse.Error("Network error: ${e.message}") - } - val responseBodyText = response.bodyAsText() - Log.d(TAG, "Gemini Response Body: $responseBodyText") + val responseBodyText = response.bodyAsText() + Log.d(TAG, "Gemini Response Body: $responseBodyText") - if (response.status.value !in 200..299) { - return LlmResponse.Error("Gemini API error: ${response.status} - $responseBodyText") - } + if (response.status.value !in 200..299) { + return LlmResponse.Error("Gemini API error: ${response.status} - $responseBodyText") + } - val jsonResponse = Json.parseToJsonElement(responseBodyText).jsonObject + val jsonResponse = Json.parseToJsonElement(responseBodyText).jsonObject - val newInteractionId = - jsonResponse[KEY_ID]?.jsonPrimitive?.content - ?: return LlmResponse.Error("Missing interaction ID in response") - val steps = jsonResponse[KEY_STEPS]?.jsonArray + val newInteractionId = + jsonResponse[KEY_ID]?.jsonPrimitive?.content + ?: return LlmResponse.Error("Missing interaction ID in response") + val steps = jsonResponse[KEY_STEPS]?.jsonArray - val parts = mutableListOf() + val parts = mutableListOf() - if (steps != null) { - for (step in steps) { - val stepObj = step.jsonObject - val stepType = stepObj[KEY_TYPE]?.jsonPrimitive?.content + if (steps != null) { + for (step in steps) { + val stepObj = step.jsonObject + val stepType = stepObj[KEY_TYPE]?.jsonPrimitive?.content - if (stepType == VALUE_MODEL_OUTPUT) { - val content = stepObj[KEY_CONTENT]?.jsonArray - if (content != null) { - for (part in content) { - val partObj = part.jsonObject - val partType = partObj[KEY_TYPE]?.jsonPrimitive?.content - if (partType == VALUE_TEXT) { - val text = partObj[KEY_TEXT]?.jsonPrimitive?.content - if (text != null) { - parts.add(LlmResponsePart.Text(text)) + if (stepType == VALUE_MODEL_OUTPUT) { + val content = stepObj[KEY_CONTENT]?.jsonArray + if (content != null) { + for (part in content) { + val partObj = part.jsonObject + val partType = partObj[KEY_TYPE]?.jsonPrimitive?.content + if (partType == VALUE_TEXT) { + val text = partObj[KEY_TEXT]?.jsonPrimitive?.content + if (text != null) { + parts.add(LlmResponsePart.Text(text)) + } } } } - } - } else if (stepType == VALUE_FUNCTION_CALL) { - val name = - stepObj[KEY_NAME]?.jsonPrimitive?.content - ?: return LlmResponse.Error("Called function without a name") - val matchingTool = - tools.find { tool -> toolConverter.getToolName(tool) == name } - ?: return LlmResponse.Error("Called unknown function: $name") + } else if (stepType == VALUE_FUNCTION_CALL) { + val name = + stepObj[KEY_NAME]?.jsonPrimitive?.content + ?: return LlmResponse.Error("Called function without a name") + val matchingTool = + tools.find { tool -> toolConverter.getToolName(tool) == name } + ?: return LlmResponse.Error("Called unknown function: $name") - val packageName = matchingTool.packageName - val functionId = matchingTool.id + val packageName = matchingTool.packageName + val functionId = matchingTool.id - val args = stepObj[KEY_ARGUMENTS]?.jsonObject - val argumentsMap = mutableMapOf() - if (args != null) { - for ((key, value) in args) { - argumentsMap[key] = value.toPrimitive() + val args = stepObj[KEY_ARGUMENTS]?.jsonObject + val argumentsMap = mutableMapOf() + if (args != null) { + for ((key, value) in args) { + argumentsMap[key] = value.toPrimitive() + } } - } - val callId = - stepObj["id"]?.jsonPrimitive?.content - ?: return LlmResponse.Error( - "Function call missing call_id in response" - ) - parts.add( - LlmResponsePart.ToolCall( - packageName = packageName, - functionId = functionId, - arguments = argumentsMap, - callId = callId + val callId = + stepObj["id"]?.jsonPrimitive?.content + ?: return LlmResponse.Error( + "Function call missing call_id in response", + ) + parts.add( + LlmResponsePart.ToolCall( + packageName = packageName, + functionId = functionId, + arguments = argumentsMap, + callId = callId, + ), ) - ) - } else { - Log.d(TAG, "Unsupported step type: $stepType") + } else { + Log.d(TAG, "Unsupported step type: $stepType") + } } } - } - return LlmResponse.Success( - interactionId = newInteractionId, - parts = parts - ) - } + return LlmResponse.Success( + interactionId = newInteractionId, + parts = parts, + ) + } - private fun JsonElement.toPrimitive(): Any? { - return when (this) { - is JsonPrimitive -> { - if (this.isString) return this.content - return this.content.toBooleanStrictOrNull() - ?: this.content.toLongOrNull() - ?: this.content.toDoubleOrNull() + private fun JsonElement.toPrimitive(): Any? { + return when (this) { + is JsonPrimitive -> { + if (this.isString) return this.content + return this.content.toBooleanStrictOrNull() + ?: this.content.toLongOrNull() + ?: this.content.toDoubleOrNull() + } + is JsonObject -> this.mapValues { it.value.toPrimitive() } + is JsonArray -> this.map { it.toPrimitive() } } - is JsonObject -> this.mapValues { it.value.toPrimitive() } - is JsonArray -> this.map { it.toPrimitive() } } - } - private fun getSystemInstruction(): String { - val currentDate = LocalDate.now().toString() - return """ + private fun getSystemInstruction(): String { + val currentDate = LocalDate.now().toString() + return """ You are an AI assistant running on Android. Today's date is $currentDate. Always reply to the user using concise, friendly, natural human language. Never respond to the user with raw JSON or structured code blocks unless explicitly asked to write code. When a user asks you to generate an image, call the generateImage tool. After generateImage completes, confirm to the user in a natural sentence (for example: "I have generated that image for you."). When a user asks you to generate an image and use it in an app (for example, setting a chat wallpaper or attaching an image to a note), first call generateImage, then call the target app function passing the returned imageUri. - """.trimIndent() - } + """.trimIndent() + } - companion object { - private const val GEMINI_INTERACTIONS_URL = - "https://generativelanguage.googleapis.com/v1beta/interactions" - private const val TAG = "GeminiProvider" + companion object { + private const val GEMINI_INTERACTIONS_URL = + "https://generativelanguage.googleapis.com/v1beta/interactions" + private const val TAG = "GeminiProvider" - private const val KEY_MODEL = "model" - private const val KEY_INPUT = "input" - private const val KEY_TOOLS = "tools" - private const val KEY_TYPE = "type" - private const val VALUE_FUNCTION = "function" - private const val KEY_PREVIOUS_INTERACTION_ID = "previous_interaction_id" - private const val KEY_SYSTEM_INSTRUCTION = "system_instruction" - private const val HEADER_API_KEY = "x-goog-api-key" - private const val KEY_ID = "id" - private const val KEY_STEPS = "steps" - private const val KEY_CONTENT = "content" - private const val VALUE_MODEL_OUTPUT = "model_output" - private const val VALUE_TEXT = "text" - private const val KEY_TEXT = "text" - private const val VALUE_FUNCTION_CALL = "function_call" - private const val KEY_NAME = "name" - private const val KEY_ARGUMENTS = "arguments" - private const val HEADER_API_REVISION = "Api-Revision" - private const val VALUE_API_REVISION = "2026-05-20" + private const val KEY_MODEL = "model" + private const val KEY_INPUT = "input" + private const val KEY_TOOLS = "tools" + private const val KEY_TYPE = "type" + private const val VALUE_FUNCTION = "function" + private const val KEY_PREVIOUS_INTERACTION_ID = "previous_interaction_id" + private const val KEY_SYSTEM_INSTRUCTION = "system_instruction" + private const val HEADER_API_KEY = "x-goog-api-key" + private const val KEY_ID = "id" + private const val KEY_STEPS = "steps" + private const val KEY_CONTENT = "content" + private const val VALUE_MODEL_OUTPUT = "model_output" + private const val VALUE_TEXT = "text" + private const val KEY_TEXT = "text" + private const val VALUE_FUNCTION_CALL = "function_call" + private const val KEY_NAME = "name" + private const val KEY_ARGUMENTS = "arguments" + private const val HEADER_API_REVISION = "Api-Revision" + private const val VALUE_API_REVISION = "2026-05-20" + } } -} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt index 1dda085..cc10981 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt @@ -29,273 +29,273 @@ import androidx.appfunctions.metadata.AppFunctionObjectTypeMetadata import androidx.appfunctions.metadata.AppFunctionReferenceTypeMetadata import androidx.appfunctions.metadata.AppFunctionStringTypeMetadata import com.example.appfunctions.agent.domain.appfunction.ToolConverter -import javax.inject.Inject -import javax.inject.Singleton import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.JsonPrimitive import kotlinx.serialization.json.buildJsonArray import kotlinx.serialization.json.buildJsonObject +import javax.inject.Inject +import javax.inject.Singleton /** Implementation of [ToolConverter] for Gemini tools. */ @Singleton class GeminiToolConverter -@Inject -constructor() : ToolConverter { - override fun convert(tool: AppFunctionMetadata): JsonObject { - return convertToolToGeminiSchema(tool) - } + @Inject + constructor() : ToolConverter { + override fun convert(tool: AppFunctionMetadata): JsonObject { + return convertToolToGeminiSchema(tool) + } - override fun getToolName(tool: AppFunctionMetadata): String { - return getGeminiFunctionName(tool) - } + override fun getToolName(tool: AppFunctionMetadata): String { + return getGeminiFunctionName(tool) + } - private fun convertToolToGeminiSchema(tool: AppFunctionMetadata): JsonObject { - val combinedName = getGeminiFunctionName(tool) - return buildJsonObject { - put(KEY_NAME, JsonPrimitive(combinedName)) - put(KEY_DESCRIPTION, JsonPrimitive(tool.description)) - put( - KEY_PARAMETERS, - buildJsonObject { - put(KEY_TYPE, JsonPrimitive(VALUE_OBJECT)) - put( - KEY_PROPERTIES, - buildJsonObject { - tool.parameters.forEach { parameter -> - val typeSchema = - mapDataTypeToGeminiSchema( - parameter.dataType, - tool.components, - tool.id, - parameterName = parameter.name - ) - put( - parameter.name, - buildJsonObject { - typeSchema.forEach { (key, value) -> put(key, value) } - put( - KEY_DESCRIPTION, - JsonPrimitive(parameter.description) - ) - } - ) - } - } - ) - val requiredParams = tool.parameters.filter { it.isRequired }.map { it.name } - if (requiredParams.isNotEmpty()) { + private fun convertToolToGeminiSchema(tool: AppFunctionMetadata): JsonObject { + val combinedName = getGeminiFunctionName(tool) + return buildJsonObject { + put(KEY_NAME, JsonPrimitive(combinedName)) + put(KEY_DESCRIPTION, JsonPrimitive(tool.description)) + put( + KEY_PARAMETERS, + buildJsonObject { + put(KEY_TYPE, JsonPrimitive(VALUE_OBJECT)) put( - KEY_REQUIRED, - buildJsonArray { - requiredParams.forEach { - add( - JsonPrimitive(it) + KEY_PROPERTIES, + buildJsonObject { + tool.parameters.forEach { parameter -> + val typeSchema = + mapDataTypeToGeminiSchema( + parameter.dataType, + tool.components, + tool.id, + parameterName = parameter.name, + ) + put( + parameter.name, + buildJsonObject { + typeSchema.forEach { (key, value) -> put(key, value) } + put( + KEY_DESCRIPTION, + JsonPrimitive(parameter.description), + ) + }, ) } - } + }, ) - } - } - ) + val requiredParams = tool.parameters.filter { it.isRequired }.map { it.name } + if (requiredParams.isNotEmpty()) { + put( + KEY_REQUIRED, + buildJsonArray { + requiredParams.forEach { + add( + JsonPrimitive(it), + ) + } + }, + ) + } + }, + ) + } } - } - private fun getGeminiFunctionName(tool: AppFunctionMetadata): String { - val baseName = tool.id.replace(INVALID_NAME_CHARS, TOOL_ID_SEPARATOR) - val components = baseName.split(TOOL_ID_SEPARATOR) - var result = "" + private fun getGeminiFunctionName(tool: AppFunctionMetadata): String { + val baseName = tool.id.replace(INVALID_NAME_CHARS, TOOL_ID_SEPARATOR) + val components = baseName.split(TOOL_ID_SEPARATOR) + var result = "" - // Build from right to left, keeping the most specific parts of the package/id - for (i in components.indices.reversed()) { - val component = components[i] - val separator = if (result.isEmpty()) "" else TOOL_ID_SEPARATOR + // Build from right to left, keeping the most specific parts of the package/id + for (i in components.indices.reversed()) { + val component = components[i] + val separator = if (result.isEmpty()) "" else TOOL_ID_SEPARATOR - val newLength = result.length + separator.length + component.length - if (newLength <= MAX_NAME_LENGTH) { - result = component + separator + result - } else { - break + val newLength = result.length + separator.length + component.length + if (newLength <= MAX_NAME_LENGTH) { + result = component + separator + result + } else { + break + } } + if (result.isEmpty()) { + result = baseName.takeLast(MAX_NAME_LENGTH) + } + return result } - if (result.isEmpty()) { - result = baseName.takeLast(MAX_NAME_LENGTH) - } - return result - } - private fun mapDataTypeToGeminiSchema( - dataType: AppFunctionDataTypeMetadata, - components: AppFunctionComponentsMetadata, - functionId: String, - visitedReferences: Set = emptySet(), - parameterName: String? = null - ): JsonObject { - return when (dataType) { - is AppFunctionStringTypeMetadata -> - buildJsonObject { - put(KEY_TYPE, JsonPrimitive(VALUE_STRING)) - if (isFileReferenceParameter(parameterName)) { - put(KEY_FORMAT, JsonPrimitive(VALUE_FILE_REFERENCE)) - } - val enumValues = dataType.enumValues - if (!enumValues.isNullOrEmpty()) { - put( - KEY_ENUMS, - buildJsonArray { - for (enumValue in enumValues) { - add(JsonPrimitive(enumValue)) - } - } - ) - } - } - is AppFunctionLongTypeMetadata -> - buildJsonObject { put(KEY_TYPE, JsonPrimitive(VALUE_INTEGER)) } - is AppFunctionIntTypeMetadata -> - buildJsonObject { - put(KEY_TYPE, JsonPrimitive(VALUE_INTEGER)) - val enumValues = dataType.enumValues - if (!enumValues.isNullOrEmpty()) { - put( - KEY_ENUMS, - buildJsonArray { - for (enumValue in enumValues) { - add(JsonPrimitive(enumValue)) - } - } - ) - } - } - is AppFunctionBooleanTypeMetadata -> - buildJsonObject { put(KEY_TYPE, JsonPrimitive(VALUE_BOOLEAN)) } - is AppFunctionDoubleTypeMetadata -> - buildJsonObject { put(KEY_TYPE, JsonPrimitive(VALUE_NUMBER)) } - is AppFunctionFloatTypeMetadata -> - buildJsonObject { put(KEY_TYPE, JsonPrimitive(VALUE_NUMBER)) } - is AppFunctionArrayTypeMetadata -> - buildJsonObject { - put(KEY_TYPE, JsonPrimitive(VALUE_ARRAY)) - put( - KEY_ITEMS, - mapDataTypeToGeminiSchema( - dataType.itemType, - components, - functionId, - visitedReferences, - parameterName - ) - ) - } - is AppFunctionObjectTypeMetadata -> - if (dataType.qualifiedName == "android.net.Uri") { + private fun mapDataTypeToGeminiSchema( + dataType: AppFunctionDataTypeMetadata, + components: AppFunctionComponentsMetadata, + functionId: String, + visitedReferences: Set = emptySet(), + parameterName: String? = null, + ): JsonObject { + return when (dataType) { + is AppFunctionStringTypeMetadata -> buildJsonObject { put(KEY_TYPE, JsonPrimitive(VALUE_STRING)) - put(KEY_FORMAT, JsonPrimitive(VALUE_FILE_REFERENCE)) + if (isFileReferenceParameter(parameterName)) { + put(KEY_FORMAT, JsonPrimitive(VALUE_FILE_REFERENCE)) + } + val enumValues = dataType.enumValues + if (!enumValues.isNullOrEmpty()) { + put( + KEY_ENUMS, + buildJsonArray { + for (enumValue in enumValues) { + add(JsonPrimitive(enumValue)) + } + }, + ) + } } - } else { + is AppFunctionLongTypeMetadata -> + buildJsonObject { put(KEY_TYPE, JsonPrimitive(VALUE_INTEGER)) } + is AppFunctionIntTypeMetadata -> buildJsonObject { - put(KEY_TYPE, JsonPrimitive(VALUE_OBJECT)) + put(KEY_TYPE, JsonPrimitive(VALUE_INTEGER)) + val enumValues = dataType.enumValues + if (!enumValues.isNullOrEmpty()) { + put( + KEY_ENUMS, + buildJsonArray { + for (enumValue in enumValues) { + add(JsonPrimitive(enumValue)) + } + }, + ) + } + } + is AppFunctionBooleanTypeMetadata -> + buildJsonObject { put(KEY_TYPE, JsonPrimitive(VALUE_BOOLEAN)) } + is AppFunctionDoubleTypeMetadata -> + buildJsonObject { put(KEY_TYPE, JsonPrimitive(VALUE_NUMBER)) } + is AppFunctionFloatTypeMetadata -> + buildJsonObject { put(KEY_TYPE, JsonPrimitive(VALUE_NUMBER)) } + is AppFunctionArrayTypeMetadata -> + buildJsonObject { + put(KEY_TYPE, JsonPrimitive(VALUE_ARRAY)) put( - KEY_PROPERTIES, - buildJsonObject { - dataType.properties.forEach { (name, type) -> - put( - name, - mapDataTypeToGeminiSchema( - type, - components, - functionId, - visitedReferences, - parameterName = name - ) - ) - } - } + KEY_ITEMS, + mapDataTypeToGeminiSchema( + dataType.itemType, + components, + functionId, + visitedReferences, + parameterName, + ), ) - if (dataType.required.isNotEmpty()) { + } + is AppFunctionObjectTypeMetadata -> + if (dataType.qualifiedName == "android.net.Uri") { + buildJsonObject { + put(KEY_TYPE, JsonPrimitive(VALUE_STRING)) + put(KEY_FORMAT, JsonPrimitive(VALUE_FILE_REFERENCE)) + } + } else { + buildJsonObject { + put(KEY_TYPE, JsonPrimitive(VALUE_OBJECT)) put( - KEY_REQUIRED, - buildJsonArray { - dataType.required.forEach { name -> - add( - JsonPrimitive(name) + KEY_PROPERTIES, + buildJsonObject { + dataType.properties.forEach { (name, type) -> + put( + name, + mapDataTypeToGeminiSchema( + type, + components, + functionId, + visitedReferences, + parameterName = name, + ), ) } - } + }, ) + if (dataType.required.isNotEmpty()) { + put( + KEY_REQUIRED, + buildJsonArray { + dataType.required.forEach { name -> + add( + JsonPrimitive(name), + ) + } + }, + ) + } } } - } - is AppFunctionReferenceTypeMetadata -> { - val referenceKey = dataType.referenceDataType - if (referenceKey == "android.net.Uri") { - return buildJsonObject { - put(KEY_TYPE, JsonPrimitive(VALUE_STRING)) - put(KEY_FORMAT, JsonPrimitive(VALUE_FILE_REFERENCE)) + is AppFunctionReferenceTypeMetadata -> { + val referenceKey = dataType.referenceDataType + if (referenceKey == "android.net.Uri") { + return buildJsonObject { + put(KEY_TYPE, JsonPrimitive(VALUE_STRING)) + put(KEY_FORMAT, JsonPrimitive(VALUE_FILE_REFERENCE)) + } } - } - if (visitedReferences.contains(referenceKey)) { - Log.d( - "GeminiToolConverter", - "Circular reference detected for $referenceKey in function $functionId. Breaking cycle." + if (visitedReferences.contains(referenceKey)) { + Log.d( + "GeminiToolConverter", + "Circular reference detected for $referenceKey in function $functionId. Breaking cycle.", + ) + return buildJsonObject { put(KEY_TYPE, JsonPrimitive(VALUE_OBJECT)) } + } + val objectType = + components.dataTypes[referenceKey] + ?: throw IllegalArgumentException( + "Reference type $referenceKey not found in components for function $functionId", + ) + mapDataTypeToGeminiSchema( + objectType, + components, + functionId, + visitedReferences + referenceKey, + parameterName, ) - return buildJsonObject { put(KEY_TYPE, JsonPrimitive(VALUE_OBJECT)) } } - val objectType = - components.dataTypes[referenceKey] - ?: throw IllegalArgumentException( - "Reference type $referenceKey not found in components for function $functionId" - ) - mapDataTypeToGeminiSchema( - objectType, - components, - functionId, - visitedReferences + referenceKey, - parameterName - ) + else -> + throw IllegalArgumentException( + "Unsupported data type: $dataType for function $functionId", + ) } - else -> - throw IllegalArgumentException( - "Unsupported data type: $dataType for function $functionId" - ) } - } - private fun isFileReferenceParameter(parameterName: String?): Boolean { - if (parameterName == null) return false - if (parameterName in KNOWN_FILE_REFERENCE_PARAM_NAMES) return true - return parameterName.endsWith("Uri", ignoreCase = true) || - parameterName.endsWith("Uris", ignoreCase = true) - } + private fun isFileReferenceParameter(parameterName: String?): Boolean { + if (parameterName == null) return false + if (parameterName in KNOWN_FILE_REFERENCE_PARAM_NAMES) return true + return parameterName.endsWith("Uri", ignoreCase = true) || + parameterName.endsWith("Uris", ignoreCase = true) + } - companion object { - private const val TOOL_ID_SEPARATOR = "_" - private const val KEY_NAME = "name" - private const val KEY_DESCRIPTION = "description" - private const val KEY_PARAMETERS = "parameters" - private const val KEY_TYPE = "type" - private const val KEY_FORMAT = "format" - private const val VALUE_FILE_REFERENCE = "file_reference" - private val KNOWN_FILE_REFERENCE_PARAM_NAMES = - setOf( - "wallpaperUri", - "imageUri", - "attachmentUri", - "ringtoneUri", - "profilePictureUri", - "audioUri" - ) - private const val VALUE_OBJECT = "object" - private const val KEY_PROPERTIES = "properties" - private const val KEY_REQUIRED = "required" - private const val VALUE_STRING = "string" - private const val VALUE_INTEGER = "integer" - private const val VALUE_BOOLEAN = "boolean" - private const val VALUE_NUMBER = "number" - private const val VALUE_ARRAY = "array" - private const val KEY_ITEMS = "items" - private const val KEY_ENUMS = "enums" - private val INVALID_NAME_CHARS = Regex("[^a-zA-Z0-9_]") - private const val MAX_NAME_LENGTH = 64 + companion object { + private const val TOOL_ID_SEPARATOR = "_" + private const val KEY_NAME = "name" + private const val KEY_DESCRIPTION = "description" + private const val KEY_PARAMETERS = "parameters" + private const val KEY_TYPE = "type" + private const val KEY_FORMAT = "format" + private const val VALUE_FILE_REFERENCE = "file_reference" + private val KNOWN_FILE_REFERENCE_PARAM_NAMES = + setOf( + "wallpaperUri", + "imageUri", + "attachmentUri", + "ringtoneUri", + "profilePictureUri", + "audioUri", + ) + private const val VALUE_OBJECT = "object" + private const val KEY_PROPERTIES = "properties" + private const val KEY_REQUIRED = "required" + private const val VALUE_STRING = "string" + private const val VALUE_INTEGER = "integer" + private const val VALUE_BOOLEAN = "boolean" + private const val VALUE_NUMBER = "number" + private const val VALUE_ARRAY = "array" + private const val KEY_ITEMS = "items" + private const val KEY_ENUMS = "enums" + private val INVALID_NAME_CHARS = Regex("[^a-zA-Z0-9_]") + private const val MAX_NAME_LENGTH = 64 + } } -} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/db/entities/MessageEntity.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/db/entities/MessageEntity.kt index 7144b03..32022ba 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/db/entities/MessageEntity.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/db/entities/MessageEntity.kt @@ -26,15 +26,15 @@ import kotlinx.serialization.json.Json @Entity( tableName = "messages", foreignKeys = - [ - ForeignKey( - entity = ThreadEntity::class, - parentColumns = ["threadId"], - childColumns = ["threadId"], - onDelete = ForeignKey.CASCADE - ) - ], - indices = [Index("threadId")] + [ + ForeignKey( + entity = ThreadEntity::class, + parentColumns = ["threadId"], + childColumns = ["threadId"], + onDelete = ForeignKey.CASCADE, + ), + ], + indices = [Index("threadId")], ) data class MessageEntity( @PrimaryKey val messageId: String, @@ -49,24 +49,24 @@ data class MessageEntity( */ val pendingIntentId: String? = null, val targetPackageName: String? = null, - val attachments: List = emptyList() + val attachments: List = emptyList(), ) @Serializable data class MessageAttachment( val uri: String, - val mimeType: String + val mimeType: String, ) class MessageAttachmentConverter { - private val dbJson = Json { - ignoreUnknownKeys = true - encodeDefaults = true - } + private val dbJson = + Json { + ignoreUnknownKeys = true + encodeDefaults = true + } @androidx.room.TypeConverter - fun fromAttachments(attachments: List): String = - dbJson.encodeToString(attachments) + fun fromAttachments(attachments: List): String = dbJson.encodeToString(attachments) @androidx.room.TypeConverter fun toAttachments(jsonString: String?): List = @@ -80,11 +80,11 @@ class MessageAttachmentConverter { enum class MessageRole { USER, - ASSISTANT + ASSISTANT, } enum class MessageProcessingStatus { PENDING_AGENT_RESPONSE, PROCESSED, - FAILED + FAILED, } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt b/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt index bdc1410..3757f9d 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt @@ -39,8 +39,8 @@ import io.ktor.client.engine.okhttp.OkHttp import io.ktor.client.plugins.HttpTimeout import io.ktor.client.plugins.contentnegotiation.ContentNegotiation import io.ktor.serialization.kotlinx.json.json -import javax.inject.Singleton import kotlinx.serialization.json.Json +import javax.inject.Singleton val Context.settingsDataStore: DataStore by preferencesDataStore(name = "settings") @@ -58,30 +58,32 @@ abstract class DataModule { @Binds @Singleton - abstract fun bindPendingIntentRepository( - impl: InMemoryPendingIntentRepository - ): PendingIntentRepository + abstract fun bindPendingIntentRepository(impl: InMemoryPendingIntentRepository): PendingIntentRepository @Binds @Singleton abstract fun bindLlmProviderFactory( - impl: com.example.appfunctions.agent.data.LlmProviderFactoryImpl + impl: com.example.appfunctions.agent.data.LlmProviderFactoryImpl, ): com.example.appfunctions.agent.domain.LlmProviderFactory companion object { @Provides @Singleton - fun provideDataStore(@ApplicationContext context: Context): DataStore { + fun provideDataStore( + @ApplicationContext context: Context, + ): DataStore { return context.settingsDataStore } @Provides @Singleton - fun provideAppDatabase(@ApplicationContext context: Context): AppDatabase { + fun provideAppDatabase( + @ApplicationContext context: Context, + ): AppDatabase { return Room.databaseBuilder( context, AppDatabase::class.java, - "app_database" + "app_database", ) .fallbackToDestructiveMigration() .build() @@ -103,7 +105,7 @@ abstract class DataModule { ignoreUnknownKeys = true prettyPrint = true isLenient = true - } + }, ) } install(HttpTimeout) { socketTimeoutMillis = 30000 } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt index 8fd76ef..22b6b6e 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt @@ -48,12 +48,6 @@ import com.example.appfunctions.agent.domain.chat.UpdateThreadParams import com.example.appfunctions.agent.domain.chat.UpdateThreadUseCase import com.example.appfunctions.agent.domain.pendingintent.SavePendingIntentUseCase import dagger.hilt.android.qualifiers.ApplicationContext -import java.io.File -import java.net.HttpURLConnection -import java.net.URL -import java.util.UUID -import javax.inject.Inject -import javax.inject.Singleton import kotlinx.coroutines.CancellationException import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.coroutineScope @@ -66,574 +60,589 @@ import kotlinx.coroutines.flow.first import kotlinx.coroutines.flow.stateIn import kotlinx.coroutines.withContext import org.json.JSONObject +import java.io.File +import java.net.HttpURLConnection +import java.net.URL +import java.util.UUID +import javax.inject.Inject +import javax.inject.Singleton /** Orchestrates the interaction between the LLM, AppFunctions, and the chat repository. */ @Singleton class AgentOrchestrator -@Inject -constructor( - @ApplicationContext private val context: Context, - private val manageThreadsUseCase: ManageThreadsUseCase, - private val observePendingMessagesUseCase: ObservePendingMessagesUseCase, - private val sendMessageUseCase: SendMessageUseCase, - private val updateMessageUseCase: UpdateMessageUseCase, - private val updateThreadUseCase: UpdateThreadUseCase, - private val llmProviderFactory: LlmProviderFactory, - private val settingsRepository: SettingsRepository, - private val getAppFunctionsUseCase: GetAppFunctionsUseCase, - private val convertInputToAppFunctionDataUseCase: ConvertInputToAppFunctionDataUseCase, - private val executeAppFunctionUseCase: ExecuteAppFunctionUseCase, - private val savePendingIntentUseCase: SavePendingIntentUseCase -) { - private val _status = MutableStateFlow(AgentStatus.Idle) - - /** The current status of the agent. */ - val status: StateFlow = _status.asStateFlow() - - /** - * Observes messages for a specific thread and processes any pending agent responses. This - * method suspends and collects messages until cancelled. - */ - suspend fun observeAndProcessMessages(threadId: String) = coroutineScope { - val threadStateFlow = - manageThreadsUseCase.getThread( - threadId - ).stateIn(this, SharingStarted.Eagerly, null) - - observePendingMessagesUseCase(threadId).collect { message -> - if (message != null) { - val thread = threadStateFlow.filterNotNull().first() - processMessage(message, thread) + @Inject + constructor( + @ApplicationContext private val context: Context, + private val manageThreadsUseCase: ManageThreadsUseCase, + private val observePendingMessagesUseCase: ObservePendingMessagesUseCase, + private val sendMessageUseCase: SendMessageUseCase, + private val updateMessageUseCase: UpdateMessageUseCase, + private val updateThreadUseCase: UpdateThreadUseCase, + private val llmProviderFactory: LlmProviderFactory, + private val settingsRepository: SettingsRepository, + private val getAppFunctionsUseCase: GetAppFunctionsUseCase, + private val convertInputToAppFunctionDataUseCase: ConvertInputToAppFunctionDataUseCase, + private val executeAppFunctionUseCase: ExecuteAppFunctionUseCase, + private val savePendingIntentUseCase: SavePendingIntentUseCase, + ) { + private val _status = MutableStateFlow(AgentStatus.Idle) + + /** The current status of the agent. */ + val status: StateFlow = _status.asStateFlow() + + /** + * Observes messages for a specific thread and processes any pending agent responses. This + * method suspends and collects messages until cancelled. + */ + suspend fun observeAndProcessMessages(threadId: String) = + coroutineScope { + val threadStateFlow = + manageThreadsUseCase.getThread( + threadId, + ).stateIn(this, SharingStarted.Eagerly, null) + + observePendingMessagesUseCase(threadId).collect { message -> + if (message != null) { + val thread = threadStateFlow.filterNotNull().first() + processMessage(message, thread) + } + } } - } - } - private suspend fun processMessage(message: MessageEntity, thread: ThreadEntity) { - _status.value = AgentStatus.Thinking + private suspend fun processMessage( + message: MessageEntity, + thread: ThreadEntity, + ) { + _status.value = AgentStatus.Thinking - try { - val provider = thread.llmModel.providerName - val apiKey = getApiKey(provider) - if (apiKey == null) { + try { + val provider = thread.llmModel.providerName + val apiKey = getApiKey(provider) + if (apiKey == null) { + completeMessageWithError( + message.messageId, + message.threadId, + "API key is missing for ${provider.name}", + ) + _status.value = AgentStatus.Idle + return + } + + val disconnectedApps = settingsRepository.disconnectedApps.first() + val allTools = getAppFunctionsUseCase().first().values.flatten() + + val targetPackageName = message.targetPackageName + val queryText = message.textContent + + val tools = filterTools(allTools, disconnectedApps, targetPackageName) + + runInteractionLoop( + message = message, + thread = thread, + apiKey = apiKey, + tools = tools, + initialInput = queryText, + ) + + _status.value = AgentStatus.Idle + } catch (e: Exception) { + Log.e("AgentOrchestrator", "Error processing message", e) completeMessageWithError( message.messageId, message.threadId, - "API key is missing for ${provider.name}" + e.message ?: "Unknown error occurred", ) _status.value = AgentStatus.Idle - return } - - val disconnectedApps = settingsRepository.disconnectedApps.first() - val allTools = getAppFunctionsUseCase().first().values.flatten() - - val targetPackageName = message.targetPackageName - val queryText = message.textContent - - val tools = filterTools(allTools, disconnectedApps, targetPackageName) - - runInteractionLoop( - message = message, - thread = thread, - apiKey = apiKey, - tools = tools, - initialInput = queryText - ) - - _status.value = AgentStatus.Idle - } catch (e: Exception) { - Log.e("AgentOrchestrator", "Error processing message", e) - completeMessageWithError( - message.messageId, - message.threadId, - e.message ?: "Unknown error occurred" - ) - _status.value = AgentStatus.Idle } - } - private fun filterTools( - allTools: List, - disconnectedApps: Set, - targetPackageName: String? - ): List { - return allTools - .filter { metadata -> - metadata.isEnabled && - metadata.packageName !in disconnectedApps && - (targetPackageName == null || metadata.packageName == targetPackageName) - } - .sortedByDescending { metadata -> metadata.id.startsWith(metadata.packageName) } - .distinctBy { metadata -> metadata.id } - } + private fun filterTools( + allTools: List, + disconnectedApps: Set, + targetPackageName: String?, + ): List { + return allTools + .filter { metadata -> + metadata.isEnabled && + metadata.packageName !in disconnectedApps && + (targetPackageName == null || metadata.packageName == targetPackageName) + } + .sortedByDescending { metadata -> metadata.id.startsWith(metadata.packageName) } + .distinctBy { metadata -> metadata.id } + } - private suspend fun runInteractionLoop( - message: MessageEntity, - thread: ThreadEntity, - apiKey: String, - tools: List, - initialInput: String - ) { - val provider = thread.llmModel.providerName - val modelName = thread.llmModel.modelName - val llmProvider = llmProviderFactory.getProvider(provider) - - var previousInteractionId = thread.latestInteractionId - var currentToolOutputs = emptyList() - var continueLoop = true - var currentInput = initialInput - val capturedAttachments = mutableListOf() - - while (continueLoop) { - val llmInput = prepareLlmInput(currentToolOutputs, currentInput) - - currentToolOutputs = emptyList() - val response = - llmProvider.generateResponse( - previousInteractionId = previousInteractionId, - input = llmInput, - tools = tools, - apiKey = apiKey, - modelName = modelName - ) + private suspend fun runInteractionLoop( + message: MessageEntity, + thread: ThreadEntity, + apiKey: String, + tools: List, + initialInput: String, + ) { + val provider = thread.llmModel.providerName + val modelName = thread.llmModel.modelName + val llmProvider = llmProviderFactory.getProvider(provider) + + var previousInteractionId = thread.latestInteractionId + var currentToolOutputs = emptyList() + var continueLoop = true + var currentInput = initialInput + val capturedAttachments = mutableListOf() + + while (continueLoop) { + val llmInput = prepareLlmInput(currentToolOutputs, currentInput) + + currentToolOutputs = emptyList() + val response = + llmProvider.generateResponse( + previousInteractionId = previousInteractionId, + input = llmInput, + tools = tools, + apiKey = apiKey, + modelName = modelName, + ) - when ( - val handleResult = - handleLlmResponse(response, message, tools, capturedAttachments) - ) { - is HandleResult.Continue -> { - currentToolOutputs = handleResult.toolOutputs - previousInteractionId = handleResult.interactionId - } + when ( + val handleResult = + handleLlmResponse(response, message, tools, capturedAttachments) + ) { + is HandleResult.Continue -> { + currentToolOutputs = handleResult.toolOutputs + previousInteractionId = handleResult.interactionId + } - is HandleResult.Stop -> { - continueLoop = false + is HandleResult.Stop -> { + continueLoop = false + } } } } - } - private suspend fun getApiKey(provider: LlmProviderName): String? { - return when (provider) { - LlmProviderName.GEMINI -> settingsRepository.geminiApiKey.first() + private suspend fun getApiKey(provider: LlmProviderName): String? { + return when (provider) { + LlmProviderName.GEMINI -> settingsRepository.geminiApiKey.first() + } } - } - private fun prepareLlmInput( - currentToolOutputs: List, - currentInput: String - ): LlmInput { - return if (currentToolOutputs.isNotEmpty()) { - LlmInput.ToolResponse(currentToolOutputs) - } else { - LlmInput.UserMessage(currentInput) + private fun prepareLlmInput( + currentToolOutputs: List, + currentInput: String, + ): LlmInput { + return if (currentToolOutputs.isNotEmpty()) { + LlmInput.ToolResponse(currentToolOutputs) + } else { + LlmInput.UserMessage(currentInput) + } } - } - private sealed class HandleResult { - data class Continue(val toolOutputs: List, val interactionId: String) : - HandleResult() + private sealed class HandleResult { + data class Continue(val toolOutputs: List, val interactionId: String) : + HandleResult() - object Stop : HandleResult() - } + object Stop : HandleResult() + } - private sealed class ExecuteToolCallsResult { - data class Success(val toolOutputs: List) : ExecuteToolCallsResult() + private sealed class ExecuteToolCallsResult { + data class Success(val toolOutputs: List) : ExecuteToolCallsResult() - data class PendingIntentAction( - val pendingIntentId: String, - val pendingIntent: PendingIntent - ) : ExecuteToolCallsResult() + data class PendingIntentAction( + val pendingIntentId: String, + val pendingIntent: PendingIntent, + ) : ExecuteToolCallsResult() - object Error : ExecuteToolCallsResult() - } + object Error : ExecuteToolCallsResult() + } - private suspend fun handleLlmResponse( - response: LlmResponse, - message: MessageEntity, - tools: List, - capturedAttachments: MutableList - ): HandleResult { - return when (response) { - is LlmResponse.Success -> { - updateThreadUseCase( - message.threadId, - UpdateThreadParams(interactionId = response.interactionId) - ) - updateMessageUseCase( - message.messageId, - UpdateMessageParams(status = MessageProcessingStatus.PROCESSED) - ) + private suspend fun handleLlmResponse( + response: LlmResponse, + message: MessageEntity, + tools: List, + capturedAttachments: MutableList, + ): HandleResult { + return when (response) { + is LlmResponse.Success -> { + updateThreadUseCase( + message.threadId, + UpdateThreadParams(interactionId = response.interactionId), + ) + updateMessageUseCase( + message.messageId, + UpdateMessageParams(status = MessageProcessingStatus.PROCESSED), + ) - val toolCalls = response.parts.filterIsInstance() - val textParts = response.parts.filterIsInstance() - - val textContent = textParts.joinToString("\n") { it.text } - - if (toolCalls.isNotEmpty()) { - when (val toolResult = executeToolCalls(toolCalls, tools, message)) { - is ExecuteToolCallsResult.Success -> { - for (output in toolResult.toolOutputs) { - runCatching { - val json = JSONObject(output.result) - val uri = json.optString("imageUri") - if (uri.isNotBlank()) { - val mimeType = - json.optString("mimeType") - .takeIf { it.isNotBlank() } - ?: "image/png" - capturedAttachments.add( - MessageAttachment(uri = uri, mimeType = mimeType) - ) + val toolCalls = response.parts.filterIsInstance() + val textParts = response.parts.filterIsInstance() + + val textContent = textParts.joinToString("\n") { it.text } + + if (toolCalls.isNotEmpty()) { + when (val toolResult = executeToolCalls(toolCalls, tools, message)) { + is ExecuteToolCallsResult.Success -> { + for (output in toolResult.toolOutputs) { + runCatching { + val json = JSONObject(output.result) + val uri = json.optString("imageUri") + if (uri.isNotBlank()) { + val mimeType = + json.optString("mimeType") + .takeIf { it.isNotBlank() } + ?: "image/png" + capturedAttachments.add( + MessageAttachment(uri = uri, mimeType = mimeType), + ) + } } } + if (textContent.isNotEmpty()) { + sendMessageUseCase( + threadId = message.threadId, + role = MessageRole.ASSISTANT, + textContent = textContent, + processingStatus = MessageProcessingStatus.PROCESSED, + ) + } + HandleResult.Continue( + toolResult.toolOutputs, + response.interactionId, + ) } - if (textContent.isNotEmpty()) { + + is ExecuteToolCallsResult.PendingIntentAction -> { + savePendingIntentUseCase( + toolResult.pendingIntentId, + toolResult.pendingIntent, + ) sendMessageUseCase( threadId = message.threadId, role = MessageRole.ASSISTANT, textContent = textContent, - processingStatus = MessageProcessingStatus.PROCESSED + processingStatus = MessageProcessingStatus.PROCESSED, + pendingIntentId = toolResult.pendingIntentId, ) + HandleResult.Stop } - HandleResult.Continue( - toolResult.toolOutputs, - response.interactionId - ) - } - is ExecuteToolCallsResult.PendingIntentAction -> { - savePendingIntentUseCase( - toolResult.pendingIntentId, - toolResult.pendingIntent - ) + is ExecuteToolCallsResult.Error -> { + HandleResult.Stop + } + } + } else { + if (textContent.isNotEmpty() || capturedAttachments.isNotEmpty()) { sendMessageUseCase( threadId = message.threadId, role = MessageRole.ASSISTANT, textContent = textContent, processingStatus = MessageProcessingStatus.PROCESSED, - pendingIntentId = toolResult.pendingIntentId + attachments = capturedAttachments, ) - HandleResult.Stop - } - - is ExecuteToolCallsResult.Error -> { - HandleResult.Stop } + HandleResult.Stop } - } else { - if (textContent.isNotEmpty() || capturedAttachments.isNotEmpty()) { - sendMessageUseCase( - threadId = message.threadId, - role = MessageRole.ASSISTANT, - textContent = textContent, - processingStatus = MessageProcessingStatus.PROCESSED, - attachments = capturedAttachments - ) - } - HandleResult.Stop } - } - - is LlmResponse.Error -> { - Log.e("AgentOrchestrator", "LLM Error: ${response.errorMessage}") - completeMessageWithError( - message.messageId, - message.threadId, - response.errorMessage - ) - _status.value = AgentStatus.Idle - HandleResult.Stop - } - } - } - private suspend fun executeToolCalls( - toolCalls: List, - tools: List, - message: MessageEntity - ): ExecuteToolCallsResult { - val results = mutableListOf() - for (toolCall in toolCalls) { - _status.value = AgentStatus.InvokingTool(toolCall.functionId, toolCall.packageName) - - val matchingTool = - tools.find { - it.packageName == toolCall.packageName && it.id == toolCall.functionId + is LlmResponse.Error -> { + Log.e("AgentOrchestrator", "LLM Error: ${response.errorMessage}") + completeMessageWithError( + message.messageId, + message.threadId, + response.errorMessage, + ) + _status.value = AgentStatus.Idle + HandleResult.Stop } - - if (matchingTool == null) { - completeMessageWithError( - message.messageId, - message.threadId, - "Tool not found: ${toolCall.functionId}" - ) - return ExecuteToolCallsResult.Error } + } - val rawConvertedInputs = toolCall.arguments.filterValues { it != null } as Map - val convertedInputs = - try { - withContext(Dispatchers.IO) { - resolveRemoteFileReferencesRecursively( - context = context, - parametersMetadata = matchingTool.parameters, - inputs = rawConvertedInputs - ) + private suspend fun executeToolCalls( + toolCalls: List, + tools: List, + message: MessageEntity, + ): ExecuteToolCallsResult { + val results = mutableListOf() + for (toolCall in toolCalls) { + _status.value = AgentStatus.InvokingTool(toolCall.functionId, toolCall.packageName) + + val matchingTool = + tools.find { + it.packageName == toolCall.packageName && it.id == toolCall.functionId } - } catch (e: Exception) { + + if (matchingTool == null) { completeMessageWithError( message.messageId, message.threadId, - "Failed to download remote file reference: ${e.message}" + "Tool not found: ${toolCall.functionId}", ) return ExecuteToolCallsResult.Error } - grantContentUriPermissionsRecursively(context, toolCall.packageName, convertedInputs) - - val appFunctionDataResult = - withContext(Dispatchers.Default) { - convertInputToAppFunctionDataUseCase( - parameters = matchingTool.parameters, - components = matchingTool.components, - inputs = convertedInputs - ) - } - - if (appFunctionDataResult.isFailure) { - completeMessageWithError( - message.messageId, - message.threadId, - "Failed to convert arguments for ${toolCall.functionId}\n ${appFunctionDataResult.exceptionOrNull()}" - ) - return ExecuteToolCallsResult.Error - } + val rawConvertedInputs = toolCall.arguments.filterValues { it != null } as Map + val convertedInputs = + try { + withContext(Dispatchers.IO) { + resolveRemoteFileReferencesRecursively( + context = context, + parametersMetadata = matchingTool.parameters, + inputs = rawConvertedInputs, + ) + } + } catch (e: Exception) { + completeMessageWithError( + message.messageId, + message.threadId, + "Failed to download remote file reference: ${e.message}", + ) + return ExecuteToolCallsResult.Error + } - val executionResult = - executeAppFunctionUseCase( - function = matchingTool, - parameters = appFunctionDataResult.getOrThrow(), - threadId = message.threadId - ) + grantContentUriPermissionsRecursively(context, toolCall.packageName, convertedInputs) - when (executionResult) { - is ExecuteAppFunctionResult.Data -> { - results.add( - ToolOutput( - functionId = toolCall.functionId, - callId = toolCall.callId, - result = executionResult.formattedJson + val appFunctionDataResult = + withContext(Dispatchers.Default) { + convertInputToAppFunctionDataUseCase( + parameters = matchingTool.parameters, + components = matchingTool.components, + inputs = convertedInputs, ) + } + + if (appFunctionDataResult.isFailure) { + completeMessageWithError( + message.messageId, + message.threadId, + "Failed to convert arguments for ${toolCall.functionId}\n ${appFunctionDataResult.exceptionOrNull()}", ) + return ExecuteToolCallsResult.Error } - is ExecuteAppFunctionResult.PendingIntentAction -> { - val pendingIntentId = java.util.UUID.randomUUID().toString() - return ExecuteToolCallsResult.PendingIntentAction( - pendingIntentId, - executionResult.pendingIntent + val executionResult = + executeAppFunctionUseCase( + function = matchingTool, + parameters = appFunctionDataResult.getOrThrow(), + threadId = message.threadId, ) - } - is ExecuteAppFunctionResult.Error -> { - val exception = executionResult.exception - if (exception is CancellationException) { - throw exception - } - val appFunctionException = - AppFunctionExceptionFormatter.getAppFunctionException(exception) - if (appFunctionException != null) { + when (executionResult) { + is ExecuteAppFunctionResult.Data -> { results.add( ToolOutput( functionId = toolCall.functionId, callId = toolCall.callId, - result = - AppFunctionExceptionFormatter.format( - appFunctionException, - toolCall.functionId - ) - ) + result = executionResult.formattedJson, + ), ) - } else { - throw IllegalStateException( - "Tool execution failed for ${toolCall.functionId}: ${exception.message}", - exception + } + + is ExecuteAppFunctionResult.PendingIntentAction -> { + val pendingIntentId = java.util.UUID.randomUUID().toString() + return ExecuteToolCallsResult.PendingIntentAction( + pendingIntentId, + executionResult.pendingIntent, ) } + + is ExecuteAppFunctionResult.Error -> { + val exception = executionResult.exception + if (exception is CancellationException) { + throw exception + } + val appFunctionException = + AppFunctionExceptionFormatter.getAppFunctionException(exception) + if (appFunctionException != null) { + results.add( + ToolOutput( + functionId = toolCall.functionId, + callId = toolCall.callId, + result = + AppFunctionExceptionFormatter.format( + appFunctionException, + toolCall.functionId, + ), + ), + ) + } else { + throw IllegalStateException( + "Tool execution failed for ${toolCall.functionId}: ${exception.message}", + exception, + ) + } + } } } + return ExecuteToolCallsResult.Success(results) } - return ExecuteToolCallsResult.Success(results) - } - private suspend fun completeMessageWithError( - messageId: String, - threadId: String, - reason: String - ) { - updateMessageUseCase( - messageId, - UpdateMessageParams(status = MessageProcessingStatus.PROCESSED) - ) - sendMessageUseCase( - threadId = threadId, - role = MessageRole.ASSISTANT, - textContent = reason, - processingStatus = MessageProcessingStatus.FAILED - ) - } + private suspend fun completeMessageWithError( + messageId: String, + threadId: String, + reason: String, + ) { + updateMessageUseCase( + messageId, + UpdateMessageParams(status = MessageProcessingStatus.PROCESSED), + ) + sendMessageUseCase( + threadId = threadId, + role = MessageRole.ASSISTANT, + textContent = reason, + processingStatus = MessageProcessingStatus.FAILED, + ) + } - private fun resolveRemoteFileReferencesRecursively( - context: Context, - parametersMetadata: List, - inputs: Map - ): Map { - val paramMap = parametersMetadata.associateBy { it.name } - return inputs.mapValues { (key, value) -> - val paramMeta = paramMap[key] - resolveValueRecursively(context, paramMeta?.dataType, value, key) + private fun resolveRemoteFileReferencesRecursively( + context: Context, + parametersMetadata: List, + inputs: Map, + ): Map { + val paramMap = parametersMetadata.associateBy { it.name } + return inputs.mapValues { (key, value) -> + val paramMeta = paramMap[key] + resolveValueRecursively(context, paramMeta?.dataType, value, key) + } } - } - private fun resolveValueRecursively( - context: Context, - dataType: AppFunctionDataTypeMetadata?, - value: Any, - paramName: String? - ): Any { - return when (value) { - is String -> { - val shouldResolve = - isFileReferenceParameter(paramName) || isUriMetadata(dataType) - if (shouldResolve && ( - value.startsWith( - "http://" - ) || value.startsWith("https://") + private fun resolveValueRecursively( + context: Context, + dataType: AppFunctionDataTypeMetadata?, + value: Any, + paramName: String?, + ): Any { + return when (value) { + is String -> { + val shouldResolve = + isFileReferenceParameter(paramName) || isUriMetadata(dataType) + if (shouldResolve && ( + value.startsWith( + "http://", + ) || value.startsWith("https://") ) - ) { - downloadRemoteFileToContentUri(context, value) - } else { - value + ) { + downloadRemoteFileToContentUri(context, value) + } else { + value + } } - } - is Map<*, *> -> { - value.entries.associate { entry -> - val k = entry.key as String - val propType = - (dataType as? AppFunctionObjectTypeMetadata)?.properties?.get(k) - k to ( - entry.value?.let { - resolveValueRecursively( - context, - propType, - it, - k - ) - } ?: "" + is Map<*, *> -> { + value.entries.associate { entry -> + val k = entry.key as String + val propType = + (dataType as? AppFunctionObjectTypeMetadata)?.properties?.get(k) + k to ( + entry.value?.let { + resolveValueRecursively( + context, + propType, + it, + k, + ) + } ?: "" ) + } } - } - is List<*> -> { - val itemType = (dataType as? AppFunctionArrayTypeMetadata)?.itemType - value.mapNotNull { item -> - item?.let { resolveValueRecursively(context, itemType, it, paramName) } + is List<*> -> { + val itemType = (dataType as? AppFunctionArrayTypeMetadata)?.itemType + value.mapNotNull { item -> + item?.let { resolveValueRecursively(context, itemType, it, paramName) } + } } + else -> value } - else -> value } - } - private fun downloadRemoteFileToContentUri(context: Context, urlString: String): String { - val url = URL(urlString) - val connection = - (url.openConnection() as HttpURLConnection).apply { - connectTimeout = 10000 - readTimeout = 15000 - } - try { - connection.connect() - val contentType = connection.contentType ?: "" - val ext = - when { - contentType.contains("png", ignoreCase = true) -> "png" - contentType.contains("jpeg", ignoreCase = true) || - contentType.contains("jpg", ignoreCase = true) -> "jpg" - contentType.contains("gif", ignoreCase = true) -> "gif" - contentType.contains("webp", ignoreCase = true) -> "webp" - urlString.substringAfterLast("/", "").contains(".") -> - urlString.substringAfterLast("/").substringAfterLast(".") - else -> "jpg" + private fun downloadRemoteFileToContentUri( + context: Context, + urlString: String, + ): String { + val url = URL(urlString) + val connection = + (url.openConnection() as HttpURLConnection).apply { + connectTimeout = 10000 + readTimeout = 15000 } - val cacheDir = File(context.cacheDir, "file_references").apply { mkdirs() } - val file = File(cacheDir, "generated_${UUID.randomUUID()}.$ext") - connection.inputStream.use { input -> - file.outputStream().use { output -> - input.copyTo(output) + try { + connection.connect() + val contentType = connection.contentType ?: "" + val ext = + when { + contentType.contains("png", ignoreCase = true) -> "png" + contentType.contains("jpeg", ignoreCase = true) || + contentType.contains("jpg", ignoreCase = true) -> "jpg" + contentType.contains("gif", ignoreCase = true) -> "gif" + contentType.contains("webp", ignoreCase = true) -> "webp" + urlString.substringAfterLast("/", "").contains(".") -> + urlString.substringAfterLast("/").substringAfterLast(".") + else -> "jpg" + } + val cacheDir = File(context.cacheDir, "file_references").apply { mkdirs() } + val file = File(cacheDir, "generated_${UUID.randomUUID()}.$ext") + connection.inputStream.use { input -> + file.outputStream().use { output -> + input.copyTo(output) + } } + val contentUri = + FileProvider.getUriForFile( + context, + "${context.packageName}.fileprovider", + file, + ) + return contentUri.toString() + } finally { + connection.disconnect() } - val contentUri = - FileProvider.getUriForFile( - context, - "${context.packageName}.fileprovider", - file - ) - return contentUri.toString() - } finally { - connection.disconnect() } - } - private fun grantContentUriPermissionsRecursively( - context: Context, - targetPackageName: String, - value: Any? - ) { - when (value) { - is String -> { - if (value.startsWith("content://")) { - runCatching { - val uri = Uri.parse(value) - context.grantUriPermission( - targetPackageName, - uri, - Intent.FLAG_GRANT_READ_URI_PERMISSION - ) + private fun grantContentUriPermissionsRecursively( + context: Context, + targetPackageName: String, + value: Any?, + ) { + when (value) { + is String -> { + if (value.startsWith("content://")) { + runCatching { + val uri = Uri.parse(value) + context.grantUriPermission( + targetPackageName, + uri, + Intent.FLAG_GRANT_READ_URI_PERMISSION, + ) + } } } - } - is Map<*, *> -> value.values.forEach { - grantContentUriPermissionsRecursively(context, targetPackageName, it) - } - is List<*> -> value.forEach { - grantContentUriPermissionsRecursively(context, targetPackageName, it) + is Map<*, *> -> + value.values.forEach { + grantContentUriPermissionsRecursively(context, targetPackageName, it) + } + is List<*> -> + value.forEach { + grantContentUriPermissionsRecursively(context, targetPackageName, it) + } } } - } - private fun isFileReferenceParameter(parameterName: String?): Boolean { - if (parameterName == null) return false - if (parameterName in KNOWN_FILE_REFERENCE_PARAM_NAMES) return true - return parameterName.endsWith("Uri", ignoreCase = true) || - parameterName.endsWith("Uris", ignoreCase = true) - } + private fun isFileReferenceParameter(parameterName: String?): Boolean { + if (parameterName == null) return false + if (parameterName in KNOWN_FILE_REFERENCE_PARAM_NAMES) return true + return parameterName.endsWith("Uri", ignoreCase = true) || + parameterName.endsWith("Uris", ignoreCase = true) + } - private fun isUriMetadata(dataType: AppFunctionDataTypeMetadata?): Boolean { - if (dataType == null) return false - if (dataType is AppFunctionObjectTypeMetadata && dataType.qualifiedName == "android.net.Uri") return true - if (dataType is AppFunctionReferenceTypeMetadata && dataType.referenceDataType == "android.net.Uri") return true - return false - } + private fun isUriMetadata(dataType: AppFunctionDataTypeMetadata?): Boolean { + if (dataType == null) return false + if (dataType is AppFunctionObjectTypeMetadata && dataType.qualifiedName == "android.net.Uri") return true + if (dataType is AppFunctionReferenceTypeMetadata && dataType.referenceDataType == "android.net.Uri") return true + return false + } - companion object { - private val KNOWN_FILE_REFERENCE_PARAM_NAMES = - setOf( - "wallpaperUri", - "imageUri", - "attachmentUri", - "ringtoneUri", - "profilePictureUri", - "audioUri" - ) + companion object { + private val KNOWN_FILE_REFERENCE_PARAM_NAMES = + setOf( + "wallpaperUri", + "imageUri", + "attachmentUri", + "ringtoneUri", + "profilePictureUri", + "audioUri", + ) + } } -} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/AppFunctionExceptionFormatter.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/AppFunctionExceptionFormatter.kt index c3b381a..fc42248 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/AppFunctionExceptionFormatter.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/AppFunctionExceptionFormatter.kt @@ -22,7 +22,10 @@ object AppFunctionExceptionFormatter { /** * Formats the given [exception] including its class name and message. */ - fun format(exception: AppFunctionException, functionId: String? = null): String { + fun format( + exception: AppFunctionException, + functionId: String? = null, + ): String { val className = exception.javaClass.simpleName val message = exception.errorMessage ?: exception.message ?: "No error message provided" val prefix = if (functionId != null) "Tool execution failed for $functionId: " else "" diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/chat/SendMessageUseCase.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/chat/SendMessageUseCase.kt index ef11560..4f9ac22 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/chat/SendMessageUseCase.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/chat/SendMessageUseCase.kt @@ -25,39 +25,39 @@ import javax.inject.Inject /** Use case to send a message in a chat thread. */ class SendMessageUseCase -@Inject -constructor( - private val chatRepository: ChatRepository -) { - /** - * Executes the use case. - * - * @param threadId The ID of the thread to send the message to. - * @param role The role of the sender (USER or ASSISTANT). - * @param textContent The content of the message. - * @param processingStatus The processing status of the message. - */ - suspend operator fun invoke( - threadId: String, - role: MessageRole, - textContent: String, - processingStatus: MessageProcessingStatus, - pendingIntentId: String? = null, - targetPackageName: String? = null, - attachments: List = emptyList() + @Inject + constructor( + private val chatRepository: ChatRepository, ) { - val message = - MessageEntity( - messageId = UUID.randomUUID().toString(), - threadId = threadId, - role = role, - textContent = textContent, - timestamp = System.currentTimeMillis(), - processingStatus = processingStatus, - pendingIntentId = pendingIntentId, - targetPackageName = targetPackageName, - attachments = attachments - ) - chatRepository.sendMessage(message) + /** + * Executes the use case. + * + * @param threadId The ID of the thread to send the message to. + * @param role The role of the sender (USER or ASSISTANT). + * @param textContent The content of the message. + * @param processingStatus The processing status of the message. + */ + suspend operator fun invoke( + threadId: String, + role: MessageRole, + textContent: String, + processingStatus: MessageProcessingStatus, + pendingIntentId: String? = null, + targetPackageName: String? = null, + attachments: List = emptyList(), + ) { + val message = + MessageEntity( + messageId = UUID.randomUUID().toString(), + threadId = threadId, + role = role, + textContent = textContent, + timestamp = System.currentTimeMillis(), + processingStatus = processingStatus, + pendingIntentId = pendingIntentId, + targetPackageName = targetPackageName, + attachments = attachments, + ) + chatRepository.sendMessage(message) + } } -} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt index 6c9758a..193e37c 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt @@ -145,7 +145,7 @@ fun AgentDemoScreen(viewModel: AgentDemoViewModel = hiltViewModel()) { fun AgentDemoContent( uiState: AgentUiState, onEvent: (AgentUiEvent) -> Unit, - initialSidePanelVisible: Boolean = false + initialSidePanelVisible: Boolean = false, ) { val context = LocalContext.current val packageManager = context.packageManager @@ -174,7 +174,7 @@ fun AgentDemoContent( drawerState = drawerState, scope = scope, packageManager = packageManager, - initialSidePanelVisible = initialSidePanelVisible + initialSidePanelVisible = initialSidePanelVisible, ) } } @@ -190,7 +190,7 @@ fun AgentDemoContent( drawerState = drawerState, drawerContent = { ModalDrawerSheet( - drawerContainerColor = MaterialTheme.colorScheme.surface + drawerContainerColor = MaterialTheme.colorScheme.surface, ) { ChatHistorySidePanel( threads = threads, @@ -198,10 +198,10 @@ fun AgentDemoContent( onEvent = { event -> onEvent(event) scope.launch { drawerState.close() } - } + }, ) } - } + }, ) { content() } @@ -224,7 +224,7 @@ fun AgentDemoLoadedScreen( drawerState: DrawerState, scope: CoroutineScope, packageManager: PackageManager, - initialSidePanelVisible: Boolean = false + initialSidePanelVisible: Boolean = false, ) { var messageText by remember { mutableStateOf(TextFieldValue("")) } var isSidePanelVisible by remember { mutableStateOf(initialSidePanelVisible) } @@ -243,13 +243,13 @@ fun AgentDemoLoadedScreen( topBar = { Row( modifier = Modifier.padding(horizontal = 8.dp, vertical = 16.dp), - verticalAlignment = Alignment.CenterVertically + verticalAlignment = Alignment.CenterVertically, ) { ModelDropdown( modifier = - Modifier - .weight(1f) - .padding(horizontal = 8.dp), + Modifier + .weight(1f) + .padding(horizontal = 8.dp), currentThread = uiState.currentThread, onModelSelected = { onEvent(AgentUiEvent.OnModelSelected(it)) }, onMenuClick = { @@ -258,39 +258,39 @@ fun AgentDemoLoadedScreen( } else { scope.launch { drawerState.open() } } - } + }, ) IconButton( onClick = { onEvent(AgentUiEvent.OnCreateThread(uiState.currentThread.llmModel)) }, - modifier = Modifier.padding(horizontal = 8.dp) + modifier = Modifier.padding(horizontal = 8.dp), ) { Icon(imageVector = Icons.Default.Add, contentDescription = "Create Thread") } } - } + }, ) { paddingValues -> Row( modifier = - Modifier - .fillMaxSize() - .imePadding() - .padding( - top = paddingValues.calculateTopPadding() - ) + Modifier + .fillMaxSize() + .imePadding() + .padding( + top = paddingValues.calculateTopPadding(), + ), ) { // Side Panel (only for wide screens) if (isWideScreen) { AnimatedVisibility( visible = isSidePanelVisible, enter = slideInHorizontally() + expandHorizontally(), - exit = slideOutHorizontally() + shrinkHorizontally() + exit = slideOutHorizontally() + shrinkHorizontally(), ) { ChatHistorySidePanel( threads = uiState.threads, currentThread = uiState.currentThread, - onEvent = onEvent + onEvent = onEvent, ) } } @@ -298,20 +298,20 @@ fun AgentDemoLoadedScreen( // Main Chat Area Column( modifier = - Modifier - .weight(1f) - .fillMaxHeight() - .padding(start = 16.dp, end = 16.dp), - verticalArrangement = Arrangement.SpaceBetween + Modifier + .weight(1f) + .fillMaxHeight() + .padding(start = 16.dp, end = 16.dp), + verticalArrangement = Arrangement.SpaceBetween, ) { // Messages List LazyColumn( modifier = - Modifier - .weight(1f) - .fillMaxWidth() - .clip(RoundedCornerShape(16.dp)), - reverseLayout = true + Modifier + .weight(1f) + .fillMaxWidth() + .clip(RoundedCornerShape(16.dp)), + reverseLayout = true, ) { // Status item at the bottom (above input) if not // idle @@ -319,21 +319,21 @@ fun AgentDemoLoadedScreen( item { StatusIndicator( status = uiState.status, - packageManager = packageManager + packageManager = packageManager, ) } } items( items = uiState.messages.reversed(), - key = { message -> message.messageId } + key = { message -> message.messageId }, ) { message -> MessageBubble( message = message, isValidAction = - message.pendingIntentId in uiState.activePendingActionIds, + message.pendingIntentId in uiState.activePendingActionIds, installedApps = uiState.installedApps, - onConfirmAction = { onEvent(AgentUiEvent.OnConfirmAction(it)) } + onConfirmAction = { onEvent(AgentUiEvent.OnConfirmAction(it)) }, ) } } @@ -378,12 +378,12 @@ fun AgentDemoLoadedScreen( anchorBounds: IntRect, windowSize: IntSize, layoutDirection: LayoutDirection, - popupContentSize: IntSize + popupContentSize: IntSize, ): IntOffset { val gap = with(density) { 2.dp.roundToPx() } return IntOffset( x = anchorBounds.left, - y = anchorBounds.top - popupContentSize.height - gap + y = anchorBounds.top - popupContentSize.height - gap, ) } } @@ -414,61 +414,61 @@ fun AgentDemoLoadedScreen( } }, modifier = - Modifier - .fillMaxWidth() - .padding(vertical = 16.dp) - .onPreviewKeyEvent { keyEvent -> - if ( - (keyEvent.key == Key.Enter || keyEvent.key == Key.NumPadEnter) && - keyEvent.type == KeyEventType.KeyDown - ) { - sendMessage() - true - } else { - false - } - }, + Modifier + .fillMaxWidth() + .padding(vertical = 16.dp) + .onPreviewKeyEvent { keyEvent -> + if ( + (keyEvent.key == Key.Enter || keyEvent.key == Key.NumPadEnter) && + keyEvent.type == KeyEventType.KeyDown + ) { + sendMessage() + true + } else { + false + } + }, enabled = uiState.status == AgentStatus.Idle, shape = CircleShape, placeholder = { Text(stringResource(R.string.agent_demo_ask_agent)) }, visualTransformation = visualTransformation, colors = - OutlinedTextFieldDefaults.colors( - unfocusedContainerColor = MaterialTheme.colorScheme.surfaceBright, - focusedContainerColor = MaterialTheme.colorScheme.surfaceBright, - unfocusedBorderColor = Color.Transparent, - focusedBorderColor = Color.Transparent - ), + OutlinedTextFieldDefaults.colors( + unfocusedContainerColor = MaterialTheme.colorScheme.surfaceBright, + focusedContainerColor = MaterialTheme.colorScheme.surfaceBright, + unfocusedBorderColor = Color.Transparent, + focusedBorderColor = Color.Transparent, + ), trailingIcon = { IconButton( onClick = sendMessage, enabled = - messageText.text.isNotBlank() && - uiState.status == AgentStatus.Idle + messageText.text.isNotBlank() && + uiState.status == AgentStatus.Idle, ) { Icon( imageVector = Icons.AutoMirrored.Filled.Send, contentDescription = - stringResource(R.string.agent_demo_send) + stringResource(R.string.agent_demo_send), ) } - } + }, ) if (showAutocomplete && filteredApps.isNotEmpty()) { Popup( popupPositionProvider = popupPositionProvider, onDismissRequest = {}, - properties = PopupProperties(focusable = false) + properties = PopupProperties(focusable = false), ) { Card( modifier = Modifier.fillMaxWidth(0.9f), elevation = CardDefaults.cardElevation(defaultElevation = 8.dp), colors = - CardDefaults.cardColors( - containerColor = MaterialTheme.colorScheme.surfaceBright - ), - shape = MaterialTheme.shapes.medium + CardDefaults.cardColors( + containerColor = MaterialTheme.colorScheme.surfaceBright, + ), + shape = MaterialTheme.shapes.medium, ) { Column(modifier = Modifier.padding(vertical = 4.dp)) { filteredApps.take(5).forEach { app -> @@ -479,18 +479,18 @@ fun AgentDemoLoadedScreen( val selectionStart = messageText.selection.start val textBeforeCursor = currentText.take( - selectionStart + selectionStart, ) val textAfterCursor = currentText.drop( - selectionStart + selectionStart, ) val mentionIndex = textBeforeCursor.lastIndexOf('@') if (mentionIndex >= 0) { val textBeforeMention = textBeforeCursor.substring( 0, - mentionIndex + mentionIndex, ) val newText = "$textBeforeMention@${app.label} $textAfterCursor" @@ -500,13 +500,13 @@ fun AgentDemoLoadedScreen( TextFieldValue( text = newText, selection = - TextRange( - newCursorPosition - ) + TextRange( + newCursorPosition, + ), ) selectedAppPackageName = app.packageName } - } + }, ) } } @@ -525,19 +525,19 @@ fun ModelDropdown( modifier: Modifier = Modifier, currentThread: ThreadEntity?, onModelSelected: (LlmModel) -> Unit, - onMenuClick: () -> Unit + onMenuClick: () -> Unit, ) { var expanded by remember { mutableStateOf(false) } ExposedDropdownMenuBox( modifier = modifier, expanded = expanded, - onExpandedChange = { expanded = !expanded } + onExpandedChange = { expanded = !expanded }, ) { Surface( modifier = Modifier.padding(bottom = 8.dp), shadowElevation = 2.dp, shape = CircleShape, - color = MaterialTheme.colorScheme.surfaceBright + color = MaterialTheme.colorScheme.surfaceBright, ) { val text = currentThread?.llmModel?.modelName @@ -551,38 +551,38 @@ fun ModelDropdown( Row( modifier = - Modifier - .fillMaxWidth() - .height(56.dp) - .padding(start = 4.dp, end = 16.dp), - verticalAlignment = Alignment.CenterVertically + Modifier + .fillMaxWidth() + .height(56.dp) + .padding(start = 4.dp, end = 16.dp), + verticalAlignment = Alignment.CenterVertically, ) { IconButton(onClick = onMenuClick) { Icon(imageVector = Icons.Default.Menu, contentDescription = "Menu") } Row( modifier = - Modifier - .weight(1f) - .fillMaxHeight() - .menuAnchor( - ExposedDropdownMenuAnchorType.PrimaryEditable, - enabled = true - ), - verticalAlignment = Alignment.CenterVertically + Modifier + .weight(1f) + .fillMaxHeight() + .menuAnchor( + ExposedDropdownMenuAnchorType.PrimaryEditable, + enabled = true, + ), + verticalAlignment = Alignment.CenterVertically, ) { Column(modifier = Modifier.weight(1f)) { Text( text = stringResource(R.string.agent_demo_title), style = MaterialTheme.typography.labelMedium, - color = MaterialTheme.colorScheme.onSurfaceVariant + color = MaterialTheme.colorScheme.onSurfaceVariant, ) Text( text = text, style = MaterialTheme.typography.bodyMedium, color = textColor, maxLines = 1, - overflow = TextOverflow.Ellipsis + overflow = TextOverflow.Ellipsis, ) } Icon(imageVector = Icons.Default.ArrowDropDown, contentDescription = null) @@ -595,21 +595,21 @@ fun ModelDropdown( onDismissRequest = { expanded = false }, modifier = Modifier.exposedDropdownSize(), containerColor = MaterialTheme.colorScheme.surfaceBright, - shape = RoundedCornerShape(28.dp) + shape = RoundedCornerShape(28.dp), ) { item { Text( "--- Gemini ---", color = MaterialTheme.colorScheme.secondary, modifier = Modifier.padding(horizontal = 24.dp, vertical = 8.dp), - style = MaterialTheme.typography.labelLarge + style = MaterialTheme.typography.labelLarge, ) } val models = listOf( LlmModel.GEMINI_3_1_PRO_PREVIEW, LlmModel.GEMINI_3_FLASH_PREVIEW, - LlmModel.GEMINI_3_1_FLASH_LITE_PREVIEW + LlmModel.GEMINI_3_1_FLASH_LITE_PREVIEW, ) items(models) { model -> DropdownMenuItem( @@ -618,7 +618,7 @@ fun ModelDropdown( onModelSelected(model) expanded = false }, - contentPadding = PaddingValues(horizontal = 24.dp, vertical = 8.dp) + contentPadding = PaddingValues(horizontal = 24.dp, vertical = 8.dp), ) } } @@ -630,7 +630,7 @@ fun MessageBubble( message: MessageEntity, isValidAction: Boolean, installedApps: List, - onConfirmAction: (String) -> Unit + onConfirmAction: (String) -> Unit, ) { val alignment = if (message.role == MessageRole.USER) Alignment.End else Alignment.Start val isError = message.processingStatus == MessageProcessingStatus.FAILED @@ -649,15 +649,15 @@ fun MessageBubble( Column( modifier = - Modifier - .fillMaxWidth() - .padding(vertical = 4.dp, horizontal = 2.dp), - horizontalAlignment = alignment + Modifier + .fillMaxWidth() + .padding(vertical = 4.dp, horizontal = 2.dp), + horizontalAlignment = alignment, ) { Surface( shape = MaterialTheme.shapes.large, color = backgroundColor, - shadowElevation = if (message.role == MessageRole.ASSISTANT) 1.dp else 0.dp + shadowElevation = if (message.role == MessageRole.ASSISTANT) 1.dp else 0.dp, ) { Column(modifier = Modifier.padding(12.dp)) { SelectionContainer { @@ -666,7 +666,7 @@ fun MessageBubble( Icon( imageVector = Icons.Filled.Warning, contentDescription = stringResource(R.string.debugging_error), - tint = textColor + tint = textColor, ) Spacer(modifier = Modifier.width(8.dp)) } @@ -701,18 +701,18 @@ fun MessageBubble( installedApps, chipBgColor, chipTextColor, - density + density, ) { val map = mutableMapOf() if (installedApps.isNotEmpty() && contentText.contains("@")) { val appLabelsPattern = installedApps.joinToString( - "|" + "|", ) { Regex.escape(it.label) } val regex = Regex( "@($appLabelsPattern)\\b", - RegexOption.IGNORE_CASE + RegexOption.IGNORE_CASE, ) regex.findAll(contentText).forEachIndexed { index, match -> val id = "chip_$index" @@ -721,17 +721,17 @@ fun MessageBubble( textMeasurer.measure( text = appName, style = - typographyStyle.copy( - fontWeight = FontWeight.Bold - ) + typographyStyle.copy( + fontWeight = FontWeight.Bold, + ), ) val widthSp = with( - density + density, ) { (measured.size.width + 8.dp.roundToPx()).toSp() } val heightSp = with( - density + density, ) { (measured.size.height + 2.dp.roundToPx()).toSp() } map[id] = @@ -739,29 +739,29 @@ fun MessageBubble( Placeholder( width = widthSp, height = heightSp, - placeholderVerticalAlign = PlaceholderVerticalAlign.TextCenter - ) + placeholderVerticalAlign = PlaceholderVerticalAlign.TextCenter, + ), ) { Surface( shape = - androidx.compose.foundation.shape.RoundedCornerShape( - 6.dp - ), - color = chipBgColor + androidx.compose.foundation.shape.RoundedCornerShape( + 6.dp, + ), + color = chipBgColor, ) { Box(contentAlignment = Alignment.Center) { Text( text = appName, color = chipTextColor, style = - typographyStyle.copy( - fontWeight = FontWeight.Bold - ), + typographyStyle.copy( + fontWeight = FontWeight.Bold, + ), modifier = - Modifier.padding( - horizontal = 4.dp, - vertical = 1.dp - ) + Modifier.padding( + horizontal = 4.dp, + vertical = 1.dp, + ), ) } } @@ -775,7 +775,7 @@ fun MessageBubble( text = formattedText, inlineContent = inlineContentMap, color = textColor, - style = typographyStyle + style = typographyStyle, ) } } @@ -788,17 +788,17 @@ fun MessageBubble( enabled = isValidAction, shape = CircleShape, colors = - ButtonDefaults.buttonColors( - containerColor = MaterialTheme.colorScheme.primary, - contentColor = MaterialTheme.colorScheme.onPrimary - ) + ButtonDefaults.buttonColors( + containerColor = MaterialTheme.colorScheme.primary, + contentColor = MaterialTheme.colorScheme.onPrimary, + ), ) { Text( if (isValidAction) { stringResource(R.string.agent_demo_confirm_action) } else { stringResource(R.string.agent_demo_action_expired) - } + }, ) } } @@ -813,11 +813,11 @@ fun MessageBubble( model = attachment.uri, contentDescription = "Generated Image", modifier = - Modifier - .fillMaxWidth(0.85f) - .height(240.dp) - .clip(RoundedCornerShape(16.dp)), - contentScale = ContentScale.Crop + Modifier + .fillMaxWidth(0.85f) + .height(240.dp) + .clip(RoundedCornerShape(16.dp)), + contentScale = ContentScale.Crop, ) } } @@ -826,21 +826,24 @@ fun MessageBubble( } @Composable -fun StatusIndicator(status: AgentStatus, packageManager: PackageManager) { +fun StatusIndicator( + status: AgentStatus, + packageManager: PackageManager, +) { when (status) { AgentStatus.Thinking -> { Row( modifier = - Modifier - .fillMaxWidth() - .padding(vertical = 8.dp), - verticalAlignment = Alignment.CenterVertically + Modifier + .fillMaxWidth() + .padding(vertical = 8.dp), + verticalAlignment = Alignment.CenterVertically, ) { CircularProgressIndicator(modifier = Modifier.size(24.dp)) Spacer(modifier = Modifier.width(8.dp)) Text( stringResource(R.string.agent_demo_thinking), - style = MaterialTheme.typography.bodyMedium + style = MaterialTheme.typography.bodyMedium, ) } } @@ -862,22 +865,22 @@ fun StatusIndicator(status: AgentStatus, packageManager: PackageManager) { Surface( modifier = - Modifier - .fillMaxWidth() - .padding(vertical = 8.dp), + Modifier + .fillMaxWidth() + .padding(vertical = 8.dp), shape = MaterialTheme.shapes.large, color = MaterialTheme.colorScheme.surfaceBright, - shadowElevation = 2.dp + shadowElevation = 2.dp, ) { Row( modifier = Modifier.padding(12.dp), - verticalAlignment = Alignment.CenterVertically + verticalAlignment = Alignment.CenterVertically, ) { appIcon?.let { Image( bitmap = it.toBitmap().asImageBitmap(), contentDescription = null, - modifier = Modifier.size(40.dp) + modifier = Modifier.size(40.dp), ) Spacer(modifier = Modifier.width(12.dp)) } @@ -888,7 +891,7 @@ fun StatusIndicator(status: AgentStatus, packageManager: PackageManager) { Spacer(modifier = Modifier.width(8.dp)) Text( stringResource(R.string.agent_demo_connecting), - style = MaterialTheme.typography.bodyMedium + style = MaterialTheme.typography.bodyMedium, ) } } @@ -907,26 +910,26 @@ fun ChatHistorySidePanel( threads: List, currentThread: ThreadEntity?, onEvent: (AgentUiEvent) -> Unit, - modifier: Modifier = Modifier + modifier: Modifier = Modifier, ) { Column( modifier = - modifier - .width(280.dp) - .fillMaxHeight() - .padding(16.dp) + modifier + .width(280.dp) + .fillMaxHeight() + .padding(16.dp), ) { Text( text = stringResource(R.string.agent_demo_chat_history), style = MaterialTheme.typography.titleLarge, color = MaterialTheme.colorScheme.onSurface, - modifier = Modifier.padding(bottom = 16.dp) + modifier = Modifier.padding(bottom = 16.dp), ) LazyColumn(modifier = Modifier.fillMaxSize()) { items( items = threads, - key = { thread -> thread.threadId } + key = { thread -> thread.threadId }, ) { thread -> val isSelected = thread.threadId == currentThread?.threadId val backgroundColor = @@ -944,26 +947,26 @@ fun ChatHistorySidePanel( Surface( modifier = - Modifier - .fillMaxWidth() - .padding(vertical = 4.dp) - .clickable { - onEvent(AgentUiEvent.OnThreadSelected(thread.threadId)) - }, + Modifier + .fillMaxWidth() + .padding(vertical = 4.dp) + .clickable { + onEvent(AgentUiEvent.OnThreadSelected(thread.threadId)) + }, shape = MaterialTheme.shapes.medium, color = backgroundColor, - contentColor = textColor + contentColor = textColor, ) { Column(modifier = Modifier.padding(12.dp)) { Text( text = thread.llmModel.modelName, style = MaterialTheme.typography.bodyMedium, - color = textColor + color = textColor, ) Text( text = "ID: ${thread.threadId.take(8)}", style = MaterialTheme.typography.bodySmall, - color = textColor.copy(alpha = 0.7f) + color = textColor.copy(alpha = 0.7f), ) } } @@ -974,7 +977,7 @@ fun ChatHistorySidePanel( class InlineAppScopingVisualTransformation( private val installedApps: List, - private val chipTextColor: Color + private val chipTextColor: Color, ) : VisualTransformation { private val regex: Regex? = if (installedApps.isNotEmpty()) { @@ -1002,8 +1005,8 @@ class InlineAppScopingVisualTransformation( withStyle( SpanStyle( color = chipTextColor, - fontWeight = FontWeight.Bold - ) + fontWeight = FontWeight.Bold, + ), ) { append(match.value) } @@ -1018,7 +1021,10 @@ class InlineAppScopingVisualTransformation( } } -fun formatMessageText(text: String, installedApps: List): AnnotatedString { +fun formatMessageText( + text: String, + installedApps: List, +): AnnotatedString { if (installedApps.isEmpty() || !text.contains("@")) { return AnnotatedString(text) } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/debugging/FunctionsFoundContent.kt b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/debugging/FunctionsFoundContent.kt index 146d1f6..ca95ccb 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/debugging/FunctionsFoundContent.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/debugging/FunctionsFoundContent.kt @@ -65,7 +65,6 @@ import androidx.compose.ui.text.font.FontWeight import androidx.compose.ui.text.withStyle import androidx.compose.ui.tooling.preview.Preview import androidx.compose.ui.unit.dp -import androidx.appfunctions.AppFunctionException import com.example.appfunctions.agent.R import com.example.appfunctions.agent.domain.appfunction.AppFunctionExceptionFormatter import com.example.appfunctions.agent.domain.appfunction.ExecuteAppFunctionResult diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/data/db/entities/MessageAttachmentConverterTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/data/db/entities/MessageAttachmentConverterTest.kt index 5c665cb..7a7994e 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/data/db/entities/MessageAttachmentConverterTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/data/db/entities/MessageAttachmentConverterTest.kt @@ -20,7 +20,6 @@ import org.junit.Assert.assertTrue import org.junit.Test class MessageAttachmentConverterTest { - private val converter = MessageAttachmentConverter() @Test @@ -29,12 +28,12 @@ class MessageAttachmentConverterTest { listOf( MessageAttachment( uri = "content://com.example.appfunctions.agent.fileprovider/cache/test.jpg", - mimeType = "image/jpeg" + mimeType = "image/jpeg", ), MessageAttachment( uri = "content://com.example.appfunctions.agent.fileprovider/cache/test.png", - mimeType = "image/png" - ) + mimeType = "image/png", + ), ) val json = converter.fromAttachments(attachments) diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt index b6f3688..e8f78da 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt @@ -22,6 +22,7 @@ import androidx.appfunctions.metadata.AppFunctionPackageMetadata import com.example.appfunctions.agent.data.LlmModel import com.example.appfunctions.agent.data.LlmProviderName import com.example.appfunctions.agent.data.SettingsRepository +import com.example.appfunctions.agent.data.db.entities.MessageAttachment import com.example.appfunctions.agent.data.db.entities.MessageEntity import com.example.appfunctions.agent.data.db.entities.MessageProcessingStatus import com.example.appfunctions.agent.data.db.entities.MessageRole @@ -86,145 +87,149 @@ class AgentOrchestratorTest { getAppFunctionsUseCase = getAppFunctionsUseCase, convertInputToAppFunctionDataUseCase = convertInputToAppFunctionDataUseCase, executeAppFunctionUseCase = executeAppFunctionUseCase, - savePendingIntentUseCase = savePendingIntentUseCase + savePendingIntentUseCase = savePendingIntentUseCase, ) } @Test - fun `observeAndProcessMessages fails when API key is missing`() = runTest { - val threadId = "thread_1" - val message = createUserMessage(threadId, "Hello", messageId = "msg_1") - val thread = createThread(threadId) + fun `observeAndProcessMessages fails when API key is missing`() = + runTest { + val threadId = "thread_1" + val message = createUserMessage(threadId, "Hello", messageId = "msg_1") + val thread = createThread(threadId) - setupDefaultMocks(threadId, message, thread, apiKey = null) + setupDefaultMocks(threadId, message, thread, apiKey = null) - agentOrchestrator.observeAndProcessMessages(threadId) + agentOrchestrator.observeAndProcessMessages(threadId) - // Verify behavior (state) - assertEquals(AgentStatus.Idle, agentOrchestrator.status.value) + // Verify behavior (state) + assertEquals(AgentStatus.Idle, agentOrchestrator.status.value) - // Verify interactions - coVerify { - updateMessageUseCase( - message.messageId, - UpdateMessageParams(status = MessageProcessingStatus.PROCESSED) - ) - sendMessageUseCase( - threadId = threadId, - role = MessageRole.ASSISTANT, - textContent = "API key is missing for GEMINI", - processingStatus = MessageProcessingStatus.FAILED - ) + // Verify interactions + coVerify { + updateMessageUseCase( + message.messageId, + UpdateMessageParams(status = MessageProcessingStatus.PROCESSED), + ) + sendMessageUseCase( + threadId = threadId, + role = MessageRole.ASSISTANT, + textContent = "API key is missing for GEMINI", + processingStatus = MessageProcessingStatus.FAILED, + ) + } } - } @Test - fun `observeAndProcessMessages fails when LLM returns error`() = runTest { - val threadId = "thread_1" - val message = createUserMessage(threadId, "Hello", messageId = "msg_1") - val thread = createThread(threadId) - val llmProvider = mockk() + fun `observeAndProcessMessages fails when LLM returns error`() = + runTest { + val threadId = "thread_1" + val message = createUserMessage(threadId, "Hello", messageId = "msg_1") + val thread = createThread(threadId) + val llmProvider = mockk() - setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) - coEvery { getAppFunctionsUseCase() } returns flowOf(emptyMap()) + setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) + coEvery { getAppFunctionsUseCase() } returns flowOf(emptyMap()) - val errorMsg = "LLM failed" - coEvery { llmProvider.generateResponse(any(), any(), any(), any(), any()) } returns - LlmResponse.Error(errorMsg) + val errorMsg = "LLM failed" + coEvery { llmProvider.generateResponse(any(), any(), any(), any(), any()) } returns + LlmResponse.Error(errorMsg) - agentOrchestrator.observeAndProcessMessages(threadId) + agentOrchestrator.observeAndProcessMessages(threadId) - // Verify behavior (state) - assertEquals(AgentStatus.Idle, agentOrchestrator.status.value) + // Verify behavior (state) + assertEquals(AgentStatus.Idle, agentOrchestrator.status.value) - // Verify interactions - coVerify { - updateMessageUseCase( - message.messageId, - UpdateMessageParams(status = MessageProcessingStatus.PROCESSED) - ) - sendMessageUseCase( - threadId = threadId, - role = MessageRole.ASSISTANT, - textContent = errorMsg, - processingStatus = MessageProcessingStatus.FAILED - ) + // Verify interactions + coVerify { + updateMessageUseCase( + message.messageId, + UpdateMessageParams(status = MessageProcessingStatus.PROCESSED), + ) + sendMessageUseCase( + threadId = threadId, + role = MessageRole.ASSISTANT, + textContent = errorMsg, + processingStatus = MessageProcessingStatus.FAILED, + ) + } } - } @Test - fun `observeAndProcessMessages succeeds when LLM returns text`() = runTest { - val threadId = "thread_1" - val message = createUserMessage(threadId, "Hello", messageId = "msg_1") - val thread = createThread(threadId) - val llmProvider = mockk() - - setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) - coEvery { getAppFunctionsUseCase() } returns flowOf(emptyMap()) - - val responseText = "Hi there" - coEvery { llmProvider.generateResponse(any(), any(), any(), any(), any()) } returns - LlmResponse.Success( - "interaction_123", - listOf(LlmResponsePart.Text(responseText)) - ) + fun `observeAndProcessMessages succeeds when LLM returns text`() = + runTest { + val threadId = "thread_1" + val message = createUserMessage(threadId, "Hello", messageId = "msg_1") + val thread = createThread(threadId) + val llmProvider = mockk() - agentOrchestrator.observeAndProcessMessages(threadId) + setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) + coEvery { getAppFunctionsUseCase() } returns flowOf(emptyMap()) - // Verify behavior (state) - assertEquals(AgentStatus.Idle, agentOrchestrator.status.value) + val responseText = "Hi there" + coEvery { llmProvider.generateResponse(any(), any(), any(), any(), any()) } returns + LlmResponse.Success( + "interaction_123", + listOf(LlmResponsePart.Text(responseText)), + ) - // Verify interactions - coVerify { - updateThreadUseCase(threadId, UpdateThreadParams(interactionId = "interaction_123")) - sendMessageUseCase( - threadId = threadId, - role = MessageRole.ASSISTANT, - textContent = responseText, - processingStatus = MessageProcessingStatus.PROCESSED - ) - updateMessageUseCase( - message.messageId, - UpdateMessageParams(status = MessageProcessingStatus.PROCESSED) - ) + agentOrchestrator.observeAndProcessMessages(threadId) + + // Verify behavior (state) + assertEquals(AgentStatus.Idle, agentOrchestrator.status.value) + + // Verify interactions + coVerify { + updateThreadUseCase(threadId, UpdateThreadParams(interactionId = "interaction_123")) + sendMessageUseCase( + threadId = threadId, + role = MessageRole.ASSISTANT, + textContent = responseText, + processingStatus = MessageProcessingStatus.PROCESSED, + ) + updateMessageUseCase( + message.messageId, + UpdateMessageParams(status = MessageProcessingStatus.PROCESSED), + ) + } } - } @Test - fun `observeAndProcessMessages scopes tools when targetPackageName is set`() = runTest { - val threadId = "thread_1" - val message = - createUserMessage( - threadId = threadId, - textContent = "run geo code address for n1c4ag", - targetPackageName = "com.google.android.appfunctiontestingagent" - ) - val thread = createThread(threadId) - val llmProvider = mockk() + fun `observeAndProcessMessages scopes tools when targetPackageName is set`() = + runTest { + val threadId = "thread_1" + val message = + createUserMessage( + threadId = threadId, + textContent = "run geo code address for n1c4ag", + targetPackageName = "com.google.android.appfunctiontestingagent", + ) + val thread = createThread(threadId) + val llmProvider = mockk() - val tool1 = createMockTool("com.google.android.appfunctiontestingagent", "run_geo_code") - val tool2 = - createMockTool("com.google.android.digitalwellbeing", "digital_well_being_tool") - mockAppFunctions(listOf(tool1, tool2)) + val tool1 = createMockTool("com.google.android.appfunctiontestingagent", "run_geo_code") + val tool2 = + createMockTool("com.google.android.digitalwellbeing", "digital_well_being_tool") + mockAppFunctions(listOf(tool1, tool2)) - setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) + setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) - coEvery { - llmProvider.generateResponse(any(), any(), any(), any(), any()) - } returns LlmResponse.Success("interaction_id", listOf(LlmResponsePart.Text("Success"))) + coEvery { + llmProvider.generateResponse(any(), any(), any(), any(), any()) + } returns LlmResponse.Success("interaction_id", listOf(LlmResponsePart.Text("Success"))) - agentOrchestrator.observeAndProcessMessages(threadId) + agentOrchestrator.observeAndProcessMessages(threadId) - coVerify { - llmProvider.generateResponse( - previousInteractionId = null, - input = eq(LlmInput.UserMessage("run geo code address for n1c4ag")), - tools = listOf(tool1), - apiKey = "dummy_key", - modelName = any() - ) + coVerify { + llmProvider.generateResponse( + previousInteractionId = null, + input = eq(LlmInput.UserMessage("run geo code address for n1c4ag")), + tools = listOf(tool1), + apiKey = "dummy_key", + modelName = any(), + ) + } } - } @Test fun `observeAndProcessMessages does not scope tools when targetPackageName is null`() = @@ -253,7 +258,7 @@ class AgentOrchestratorTest { input = eq(LlmInput.UserMessage("run geo code address for n1c4ag")), tools = listOf(tool1, tool2), apiKey = "dummy_key", - modelName = any() + modelName = any(), ) } } @@ -284,17 +289,18 @@ class AgentOrchestratorTest { coEvery { llmProvider.generateResponse(null, any(), any(), any(), any()) - } returns LlmResponse.Success( - "interaction_1", - listOf( - LlmResponsePart.ToolCall( - packageName = "com.example.calendar", - functionId = "create_event", - arguments = emptyMap(), - callId = "call_1", - ) + } returns + LlmResponse.Success( + "interaction_1", + listOf( + LlmResponsePart.ToolCall( + packageName = "com.example.calendar", + functionId = "create_event", + arguments = emptyMap(), + callId = "call_1", + ), + ), ) - ) val expectedErrorOutput = "Tool execution failed for create_event: Error: AppFunctionPermissionRequiredException - Calendar permission required" @@ -307,17 +313,18 @@ class AgentOrchestratorTest { coVerify { llmProvider.generateResponse( previousInteractionId = eq("interaction_1"), - input = eq( - LlmInput.ToolResponse( - listOf( - ToolOutput( - functionId = "create_event", - callId = "call_1", - result = expectedErrorOutput, - ) - ) - ) - ), + input = + eq( + LlmInput.ToolResponse( + listOf( + ToolOutput( + functionId = "create_event", + callId = "call_1", + result = expectedErrorOutput, + ), + ), + ), + ), tools = listOf(tool1), apiKey = "dummy_key", modelName = any(), @@ -329,7 +336,7 @@ class AgentOrchestratorTest { threadId: String, textContent: String, messageId: String = "message_1", - targetPackageName: String? = null + targetPackageName: String? = null, ) = MessageEntity( messageId = messageId, threadId = threadId, @@ -337,24 +344,24 @@ class AgentOrchestratorTest { textContent = textContent, timestamp = System.currentTimeMillis(), processingStatus = MessageProcessingStatus.PENDING_AGENT_RESPONSE, - targetPackageName = targetPackageName + targetPackageName = targetPackageName, ) private fun createThread( threadId: String, llmModel: LlmModel = LlmModel.GEMINI_3_FLASH_PREVIEW, - latestInteractionId: String? = null + latestInteractionId: String? = null, ) = ThreadEntity( threadId = threadId, createdAt = System.currentTimeMillis(), llmModel = llmModel, - latestInteractionId = latestInteractionId + latestInteractionId = latestInteractionId, ) private fun createMockTool( packageName: String, id: String, - isEnabled: Boolean = true + isEnabled: Boolean = true, ): AppFunctionMetadata { val tool = mockk(relaxed = true) every { tool.packageName } returns packageName @@ -374,7 +381,7 @@ class AgentOrchestratorTest { thread: ThreadEntity, apiKey: String? = "dummy_key", disconnectedApps: Set = emptySet(), - llmProvider: LlmProvider = mockk() + llmProvider: LlmProvider = mockk(), ) { coEvery { observePendingMessagesUseCase(threadId) } returns flow { @@ -404,18 +411,18 @@ class AgentOrchestratorTest { packageName = "com.example.appfunctions.agent", functionId = "generateImage", arguments = mapOf("prompt" to "dog"), - callId = "call_1" + callId = "call_1", ) val firstResponse = LlmResponse.Success( interactionId = "interaction_1", - parts = listOf(toolCall) + parts = listOf(toolCall), ) val secondResponse = LlmResponse.Success( interactionId = "interaction_2", - parts = listOf(LlmResponsePart.Text("Here is your image!")) + parts = listOf(LlmResponsePart.Text("Here is your image!")), ) coEvery { @@ -424,7 +431,7 @@ class AgentOrchestratorTest { input = any(), tools = any(), apiKey = any(), - modelName = any() + modelName = any(), ) } returns firstResponse @@ -434,7 +441,7 @@ class AgentOrchestratorTest { input = any(), tools = any(), apiKey = any(), - modelName = any() + modelName = any(), ) } returns secondResponse @@ -447,7 +454,9 @@ class AgentOrchestratorTest { } returns ExecuteAppFunctionResult.Data( data = AppFunctionData.EMPTY, - formattedJson = """{"imageUri":"content://com.example.appfunctions.agent.fileprovider/cache/test.jpg","mimeType":"image/jpeg","prompt":"dog"}""" + formattedJson = + """{"imageUri":"content://com.example.appfunctions.agent.fileprovider""" + + """/cache/test.jpg","mimeType":"image/jpeg","prompt":"dog"}""", ) agentOrchestrator.observeAndProcessMessages(threadId) @@ -461,12 +470,14 @@ class AgentOrchestratorTest { pendingIntentId = null, targetPackageName = null, attachments = - listOf( - com.example.appfunctions.agent.data.db.entities.MessageAttachment( - uri = "content://com.example.appfunctions.agent.fileprovider/cache/test.jpg", - mimeType = "image/jpeg" - ) - ) + listOf( + MessageAttachment( + uri = + "content://com.example.appfunctions.agent.fileprovider" + + "/cache/test.jpg", + mimeType = "image/jpeg", + ), + ), ) } } @@ -489,7 +500,7 @@ class AgentOrchestratorTest { packageName = "com.example.targetapp", functionId = "setWallpaper", arguments = mapOf("uri" to contentUri), - callId = "call_2" + callId = "call_2", ) coEvery { @@ -498,12 +509,12 @@ class AgentOrchestratorTest { input = any(), tools = any(), apiKey = any(), - modelName = any() + modelName = any(), ) } returns LlmResponse.Success( interactionId = "interaction_1", - parts = listOf(toolCall) + parts = listOf(toolCall), ) coEvery { @@ -512,12 +523,12 @@ class AgentOrchestratorTest { input = any(), tools = any(), apiKey = any(), - modelName = any() + modelName = any(), ) } returns LlmResponse.Success( interactionId = "interaction_2", - parts = listOf(LlmResponsePart.Text("Wallpaper set")) + parts = listOf(LlmResponsePart.Text("Wallpaper set")), ) coEvery { @@ -529,7 +540,7 @@ class AgentOrchestratorTest { } returns ExecuteAppFunctionResult.Data( data = AppFunctionData.EMPTY, - formattedJson = """{"success":true}""" + formattedJson = """{"success":true}""", ) agentOrchestrator.observeAndProcessMessages(threadId) @@ -538,7 +549,7 @@ class AgentOrchestratorTest { context.grantUriPermission( eq("com.example.targetapp"), any(), - eq(Intent.FLAG_GRANT_READ_URI_PERMISSION) + eq(Intent.FLAG_GRANT_READ_URI_PERMISSION), ) } } diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/AppFunctionExceptionFormatterTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/AppFunctionExceptionFormatterTest.kt index b60f919..906a3dd 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/AppFunctionExceptionFormatterTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/AppFunctionExceptionFormatterTest.kt @@ -86,7 +86,8 @@ class AppFunctionExceptionFormatterTest { "com.example.chatapp.appfunctions.BaseChatAppFunctionService#send", ) assertEquals( - "Tool execution failed for com.example.chatapp.appfunctions.BaseChatAppFunctionService#send: Error: AppFunctionInvalidArgumentException - Message body cannot be empty", + "Tool execution failed for com.example.chatapp.appfunctions.BaseChatAppFunctionService#send: " + + "Error: AppFunctionInvalidArgumentException - Message body cannot be empty", formatted, ) } diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ExecuteAppFunctionUseCaseTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ExecuteAppFunctionUseCaseTest.kt index 7d4db46..9c660ff 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ExecuteAppFunctionUseCaseTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ExecuteAppFunctionUseCaseTest.kt @@ -38,38 +38,39 @@ class ExecuteAppFunctionUseCaseTest { private val convertAppFunctionDataToJsonUseCase = mockk() private val useCase = ExecuteAppFunctionUseCase(appFunctionManager, convertAppFunctionDataToJsonUseCase) - @Test - fun `invoke returns Error with original AppFunctionException when response is Error`() = runTest { - val function = mockk(relaxed = true) - every { function.packageName } returns "com.example.app" - every { function.id } returns "test_function" - val parameters = mockk() + fun `invoke returns Error with original AppFunctionException when response is Error`() = + runTest { + val function = mockk(relaxed = true) + every { function.packageName } returns "com.example.app" + every { function.id } returns "test_function" + val parameters = mockk() - val appException = AppFunctionPermissionRequiredException("Permission needed") - coEvery { appFunctionManager.executeAppFunction(any()) } returns ExecuteAppFunctionResponse.Error(appException) + val appException = AppFunctionPermissionRequiredException("Permission needed") + coEvery { appFunctionManager.executeAppFunction(any()) } returns ExecuteAppFunctionResponse.Error(appException) - val result = useCase(function, parameters) + val result = useCase(function, parameters) - assertTrue(result is ExecuteAppFunctionResult.Error) - val errorResult = result as ExecuteAppFunctionResult.Error - assertEquals(appException, errorResult.exception) - } + assertTrue(result is ExecuteAppFunctionResult.Error) + val errorResult = result as ExecuteAppFunctionResult.Error + assertEquals(appException, errorResult.exception) + } @Test - fun `invoke returns Error when executeAppFunction throws exception`() = runTest { - val function = mockk(relaxed = true) - every { function.packageName } returns "com.example.app" - every { function.id } returns "test_function" - val parameters = mockk() + fun `invoke returns Error when executeAppFunction throws exception`() = + runTest { + val function = mockk(relaxed = true) + every { function.packageName } returns "com.example.app" + every { function.id } returns "test_function" + val parameters = mockk() - val runtimeException = RuntimeException("Unexpected crash") - coEvery { appFunctionManager.executeAppFunction(any()) } throws runtimeException + val runtimeException = RuntimeException("Unexpected crash") + coEvery { appFunctionManager.executeAppFunction(any()) } throws runtimeException - val result = useCase(function, parameters) + val result = useCase(function, parameters) - assertTrue(result is ExecuteAppFunctionResult.Error) - val errorResult = result as ExecuteAppFunctionResult.Error - assertEquals(runtimeException, errorResult.exception) - } + assertTrue(result is ExecuteAppFunctionResult.Error) + val errorResult = result as ExecuteAppFunctionResult.Error + assertEquals(runtimeException, errorResult.exception) + } } From 0ce6468f7d9ef82a48d201031bca013019ec050b Mon Sep 17 00:00:00 2001 From: Caleb Areeveso Date: Tue, 14 Jul 2026 13:19:35 +0000 Subject: [PATCH 07/16] refactor: break down generateImage into private helpers, improve null safety and readability Change-Id: I584583bb8b04bb7a4530f880a6441574e125fbbd --- .../data/BaseBuiltInAppFunctionService.kt | 255 ++++++++++-------- .../agent/domain/AgentOrchestrator.kt | 2 +- 2 files changed, 141 insertions(+), 116 deletions(-) diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/BaseBuiltInAppFunctionService.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/BaseBuiltInAppFunctionService.kt index 912426e..e1a56fa 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/BaseBuiltInAppFunctionService.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/BaseBuiltInAppFunctionService.kt @@ -178,146 +178,171 @@ abstract class BaseBuiltInAppFunctionService : AppFunctionService() { aspectRatio: String? = null, ): GeneratedImageResult = withContext(Dispatchers.IO) { - val context = this@BaseBuiltInAppFunctionService - val apiKey = - context.settingsDataStore.data - .first()[stringPreferencesKey("gemini_api_key")] - ?.takeIf { it.isNotBlank() } - ?: BuildConfig.GEMINI_API_KEY.takeIf { it.isNotBlank() } - if (apiKey.isNullOrBlank()) { - throw IllegalStateException( - "Gemini API key is not configured. Please set gemini_api_key in settings.", - ) - } + val apiKey = getOrFetchApiKey() + val requestPayload = buildImageGenerationPayload(prompt, aspectRatio) + val responseText = executeImageRequest(apiKey, requestPayload) + saveBase64ImageToCache(responseText, prompt) + } - val requestJson = - JSONObject().apply { + private suspend fun getOrFetchApiKey(): String { + val apiKey = + settingsDataStore.data + .first()[stringPreferencesKey("gemini_api_key")] + ?.takeIf { it.isNotBlank() } + ?: BuildConfig.GEMINI_API_KEY.takeIf { it.isNotBlank() } + if (apiKey.isNullOrBlank()) { + throw IllegalStateException( + "Gemini API key is not configured. Please set gemini_api_key in settings.", + ) + } + return apiKey + } + + private fun buildImageGenerationPayload( + prompt: String, + aspectRatio: String?, + ): String = + JSONObject().apply { + put( + "contents", + JSONArray().apply { put( - "contents", - JSONArray().apply { + JSONObject().apply { put( - JSONObject().apply { - put( - "parts", - JSONArray().apply { - put(JSONObject().apply { put("text", prompt) }) - }, - ) + "parts", + JSONArray().apply { + put(JSONObject().apply { put("text", prompt) }) }, ) }, ) - put( - "generationConfig", - JSONObject().apply { - put("responseModalities", JSONArray().apply { put("IMAGE") }) - if (!aspectRatio.isNullOrBlank()) { - put( - "imageConfig", - JSONObject().apply { - put("aspectRatio", aspectRatio) - }, - ) - } - }, - ) - } + }, + ) + put( + "generationConfig", + JSONObject().apply { + put("responseModalities", JSONArray().apply { put("IMAGE") }) + if (!aspectRatio.isNullOrBlank()) { + put( + "imageConfig", + JSONObject().apply { + put("aspectRatio", aspectRatio) + }, + ) + } + }, + ) + }.toString() - val endpointUrl = - "https://generativelanguage.googleapis.com/v1beta/models/gemini-3.1-flash-image-preview:generateContent?key=$apiKey" - val url = URL(endpointUrl) - val connection = - (url.openConnection() as HttpURLConnection).apply { - requestMethod = "POST" - setRequestProperty("Content-Type", "application/json") - doOutput = true - connectTimeout = 30000 - readTimeout = 60000 - } + private fun executeImageRequest( + apiKey: String, + requestPayload: String, + ): String { + val endpointUrl = + "https://generativelanguage.googleapis.com/v1beta/models/gemini-3.1-flash-image-preview:generateContent?key=$apiKey" + val connection = + (URL(endpointUrl).openConnection() as HttpURLConnection).apply { + requestMethod = "POST" + setRequestProperty("Content-Type", "application/json") + doOutput = true + connectTimeout = 30000 + readTimeout = 60000 + } - try { - connection.outputStream.use { os -> - os.write(requestJson.toString().toByteArray(Charsets.UTF_8)) - } + return try { + connection.outputStream.use { os -> + os.write(requestPayload.toByteArray(Charsets.UTF_8)) + } - val responseCode = connection.responseCode - if (responseCode != HttpURLConnection.HTTP_OK) { - val errorBody = - connection.errorStream?.bufferedReader()?.use { it.readText() } - ?: "HTTP $responseCode" - throw IllegalStateException( - "Image generation failed ($responseCode): $errorBody", - ) - } + val responseCode = connection.responseCode + if (responseCode != HttpURLConnection.HTTP_OK) { + val errorBody = + connection.errorStream?.bufferedReader()?.use { it.readText() } + ?: "HTTP $responseCode" + throw IllegalStateException( + "Image generation failed ($responseCode): $errorBody", + ) + } - val responseText = connection.inputStream.bufferedReader().use { it.readText() } - val responseJson = JSONObject(responseText) - val candidates = responseJson.optJSONArray("candidates") - if (candidates == null || candidates.length() == 0) { - throw IllegalStateException( - "No candidates returned from Gemini image generation API", - ) - } + connection.inputStream.bufferedReader().use { it.readText() } + } finally { + connection.disconnect() + } + } - val parts = - candidates - .getJSONObject(0) - .optJSONObject("content") - ?.optJSONArray("parts") - if (parts == null || parts.length() == 0) { - throw IllegalStateException( - "No parts returned in candidate content. Gemini response: $responseText", - ) - } + private data class ImageInlineData(val base64: String, val mimeType: String) + + private fun saveBase64ImageToCache( + responseText: String, + prompt: String, + ): GeneratedImageResult { + val responseJson = JSONObject(responseText) + val candidates = responseJson.optJSONArray("candidates") + if (candidates == null || candidates.length() == 0) { + throw IllegalStateException( + "No candidates returned from Gemini image generation API", + ) + } + + val parts = + candidates + .getJSONObject(0) + .optJSONObject("content") + ?.optJSONArray("parts") + if (parts == null || parts.length() == 0) { + throw IllegalStateException( + "No parts returned in candidate content. Gemini response: $responseText", + ) + } - var base64Data: String? = null - var mimeType = "image/png" - for (i in 0 until parts.length()) { + val inlineResult = + (0 until parts.length()) + .asSequence() + .mapNotNull { i -> val part = parts.getJSONObject(i) val inlineData = part.optJSONObject("inlineData") ?: part.optJSONObject("inline_data") if (inlineData != null) { - base64Data = inlineData.optString("data") - val returnedMime = + val data = inlineData.optString("data") + val mime = inlineData .optString("mimeType") .takeIf { it.isNotBlank() } - ?: inlineData.optString("mime_type") - if (returnedMime.isNotBlank()) { - mimeType = returnedMime - } - break + ?: inlineData.optString("mime_type").takeIf { it.isNotBlank() } + ?: "image/png" + if (data.isNotBlank()) ImageInlineData(data, mime) else null + } else { + null } } - - if (base64Data.isNullOrBlank()) { - throw IllegalStateException( - "No inlineData image found in response parts. Gemini response: $responseText", - ) - } - - val imageBytes = Base64.decode(base64Data, Base64.DEFAULT) - val extension = - if (mimeType.contains("jpeg") || mimeType.contains("jpg")) "jpg" else "png" - val cachedFile = - File( - context.cacheDir, - "generated_${UUID.randomUUID()}.$extension", - ) - cachedFile.writeBytes(imageBytes) - - val authority = "${context.packageName}.fileprovider" - val contentUri = FileProvider.getUriForFile(context, authority, cachedFile) - GeneratedImageResult( - imageUri = contentUri.toString(), - mimeType = mimeType, - prompt = prompt, + .firstOrNull() + ?: throw IllegalStateException( + "No inlineData image found in response parts. Gemini response: $responseText", ) - } finally { - connection.disconnect() + + val imageBytes = Base64.decode(inlineResult.base64, Base64.DEFAULT) + val extension = + when { + inlineResult.mimeType.contains("jpeg", ignoreCase = true) || + inlineResult.mimeType.contains("jpg", ignoreCase = true) -> "jpg" + else -> "png" } - } + val cachedFile = + File( + cacheDir, + "generated_${UUID.randomUUID()}.$extension", + ) + cachedFile.writeBytes(imageBytes) + + val authority = "$packageName.fileprovider" + val contentUri = FileProvider.getUriForFile(this, authority, cachedFile) + return GeneratedImageResult( + imageUri = contentUri.toString(), + mimeType = inlineResult.mimeType, + prompt = prompt, + ) + } /** Represents the result of an image generation request. */ @AppFunctionSerializable diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt index 22b6b6e..aa99700 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt @@ -541,7 +541,7 @@ class AgentOrchestrator is List<*> -> { val itemType = (dataType as? AppFunctionArrayTypeMetadata)?.itemType value.mapNotNull { item -> - item?.let { resolveValueRecursively(context, itemType, it, paramName) } + if (item != null) resolveValueRecursively(context, itemType, item, paramName) else null } } else -> value From b642858edd95f1dd571d8358ee2fd00148e778c6 Mon Sep 17 00:00:00 2001 From: Caleb Areeveso Date: Wed, 15 Jul 2026 09:11:01 +0000 Subject: [PATCH 08/16] style: apply spotless formatting and rename service files to match class names Change-Id: I7557376d8765ef9761534936177a8363f5b79808 --- .../AppFunctionInstrumentationTest.kt | 1 - .../chatapp/uicomponents/ChatScreen.kt | 18 +- ChatApp/shared/build.gradle.kts | 1 - .../example/chatapp/BaseChatApplication.kt | 1 + .../BaseChatAppFunctionService.kt | 10 +- .../com/example/chatapp/appfunctions/Util.kt | 4 +- .../com/example/chatapp/util/Linkify.kt | 31 +- ChatApp/wear/build.gradle.kts | 1 - .../wear/uicomponents/WearChatScreen.kt | 7 +- agent/app/build.gradle.kts | 9 +- ...ns.kt => BaseBuiltInAppFunctionService.kt} | 276 ++++---- ...tions.kt => BaseFakeAppFunctionService.kt} | 4 +- .../agent/data/GeminiProviderImpl.kt | 350 ++++----- .../agent/data/db/entities/MessageEntity.kt | 38 +- .../appfunctions/agent/di/DataModule.kt | 20 +- .../agent/domain/AgentOrchestrator.kt | 665 +++++++++--------- .../AppFunctionExceptionFormatter.kt | 5 +- .../agent/domain/chat/SendMessageUseCase.kt | 68 +- .../ui/screens/agentdemo/AgentDemoScreen.kt | 340 ++++----- .../debugging/FunctionsFoundContent.kt | 1 - .../MessageAttachmentConverterTest.kt | 7 +- .../agent/domain/AgentOrchestratorTest.kt | 317 +++++---- .../AppFunctionExceptionFormatterTest.kt | 3 +- .../ExecuteAppFunctionUseCaseTest.kt | 51 +- 24 files changed, 1124 insertions(+), 1104 deletions(-) rename agent/app/src/main/java/com/example/appfunctions/agent/data/{BuiltInAppFunctions.kt => BaseBuiltInAppFunctionService.kt} (56%) rename agent/app/src/main/java/com/example/appfunctions/agent/data/{FakeAppFunctions.kt => BaseFakeAppFunctionService.kt} (96%) diff --git a/ChatApp/app/src/androidTest/kotlin/com/example/chatapp/appfunctions/AppFunctionInstrumentationTest.kt b/ChatApp/app/src/androidTest/kotlin/com/example/chatapp/appfunctions/AppFunctionInstrumentationTest.kt index 74e7e8e..4678127 100644 --- a/ChatApp/app/src/androidTest/kotlin/com/example/chatapp/appfunctions/AppFunctionInstrumentationTest.kt +++ b/ChatApp/app/src/androidTest/kotlin/com/example/chatapp/appfunctions/AppFunctionInstrumentationTest.kt @@ -24,7 +24,6 @@ import androidx.appfunctions.ExecuteAppFunctionRequest import androidx.appfunctions.ExecuteAppFunctionResponse import androidx.test.core.app.ApplicationProvider import androidx.test.ext.junit.runners.AndroidJUnit4 -import com.example.chatapp.appfunctions.ChatAppFunctionService import com.example.chatapp.data.MessageRepository import com.example.chatapp.data.RecipientsRepository import com.google.common.truth.Truth.assertThat diff --git a/ChatApp/app/src/main/kotlin/com/example/chatapp/uicomponents/ChatScreen.kt b/ChatApp/app/src/main/kotlin/com/example/chatapp/uicomponents/ChatScreen.kt index 388dd89..13f1b9d 100644 --- a/ChatApp/app/src/main/kotlin/com/example/chatapp/uicomponents/ChatScreen.kt +++ b/ChatApp/app/src/main/kotlin/com/example/chatapp/uicomponents/ChatScreen.kt @@ -269,14 +269,16 @@ fun MessageBubble( } } if (message.content.isNotEmpty()) { - val linkColor = if (message.isInbound) { - MaterialTheme.colorScheme.primary - } else { - MaterialTheme.colorScheme.inversePrimary - } - val annotatedText = remember(message.content, linkColor) { - linkifyString(message.content, linkColor = linkColor) - } + val linkColor = + if (message.isInbound) { + MaterialTheme.colorScheme.primary + } else { + MaterialTheme.colorScheme.inversePrimary + } + val annotatedText = + remember(message.content, linkColor) { + linkifyString(message.content, linkColor = linkColor) + } Text( modifier = Modifier.padding(16.dp), text = annotatedText, diff --git a/ChatApp/shared/build.gradle.kts b/ChatApp/shared/build.gradle.kts index b9aba68..5a720e6 100644 --- a/ChatApp/shared/build.gradle.kts +++ b/ChatApp/shared/build.gradle.kts @@ -48,5 +48,4 @@ dependencies { // App functions implementation(libs.androidx.appfunctions) ksp(libs.androidx.appfunctions.compiler) - } diff --git a/ChatApp/shared/src/main/kotlin/com/example/chatapp/BaseChatApplication.kt b/ChatApp/shared/src/main/kotlin/com/example/chatapp/BaseChatApplication.kt index b051a26..4d53914 100644 --- a/ChatApp/shared/src/main/kotlin/com/example/chatapp/BaseChatApplication.kt +++ b/ChatApp/shared/src/main/kotlin/com/example/chatapp/BaseChatApplication.kt @@ -16,4 +16,5 @@ package com.example.chatapp import android.app.Application + abstract class BaseChatApplication : Application() diff --git a/ChatApp/shared/src/main/kotlin/com/example/chatapp/appfunctions/BaseChatAppFunctionService.kt b/ChatApp/shared/src/main/kotlin/com/example/chatapp/appfunctions/BaseChatAppFunctionService.kt index caae605..19cfa52 100644 --- a/ChatApp/shared/src/main/kotlin/com/example/chatapp/appfunctions/BaseChatAppFunctionService.kt +++ b/ChatApp/shared/src/main/kotlin/com/example/chatapp/appfunctions/BaseChatAppFunctionService.kt @@ -19,19 +19,19 @@ import android.app.PendingIntent import android.content.Intent import android.net.Uri import androidx.annotation.RequiresApi +import androidx.appfunctions.AppFunction import androidx.appfunctions.AppFunctionAppUnknownException import androidx.appfunctions.AppFunctionElementNotFoundException import androidx.appfunctions.AppFunctionInvalidArgumentException import androidx.appfunctions.AppFunctionService import androidx.appfunctions.AppFunctionServiceEntryPoint import androidx.appfunctions.AppFunctionStringValueConstraint -import androidx.appfunctions.AppFunction import com.example.chatapp.data.CallManager import com.example.chatapp.data.MessageRepository import com.example.chatapp.data.RecipientsRepository import dagger.hilt.android.AndroidEntryPoint -import javax.inject.Inject import kotlinx.coroutines.CancellationException +import javax.inject.Inject /** * Service entry point for chat-related AppFunctions such as searching contacts, sending messages, and making calls. @@ -44,7 +44,9 @@ import kotlinx.coroutines.CancellationException ) abstract class BaseChatAppFunctionService : AppFunctionService() { @Inject lateinit var messageRepository: MessageRepository + @Inject lateinit var recipientsRepository: RecipientsRepository + @Inject lateinit var callManager: CallManager /** @@ -153,9 +155,7 @@ abstract class BaseChatAppFunctionService : AppFunctionService() { * @throws AppFunctionElementNotFoundException If no recipient exists for endpointValue. If thrown, call "searchContacts" to find the correct ID. */ @AppFunction(isDescribedByKDoc = true) - suspend fun makeCall( - endpointValue: String, - ): PendingIntent { + suspend fun makeCall(endpointValue: String): PendingIntent { val recipient = recipientsRepository.getRecipientById(endpointValue) ?: throw AppFunctionElementNotFoundException( diff --git a/ChatApp/shared/src/main/kotlin/com/example/chatapp/appfunctions/Util.kt b/ChatApp/shared/src/main/kotlin/com/example/chatapp/appfunctions/Util.kt index 81d0354..51ccd17 100644 --- a/ChatApp/shared/src/main/kotlin/com/example/chatapp/appfunctions/Util.kt +++ b/ChatApp/shared/src/main/kotlin/com/example/chatapp/appfunctions/Util.kt @@ -13,12 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package com.example.chatapp.appfunctions import androidx.appfunctions.AppFunctionSerializable - /** * Represents a result from a contact or group search. */ @@ -67,4 +65,4 @@ data class ChatGroup( val name: String, /** List of members belonging to the group. */ val recipients: List, -) \ No newline at end of file +) diff --git a/ChatApp/shared/src/main/kotlin/com/example/chatapp/util/Linkify.kt b/ChatApp/shared/src/main/kotlin/com/example/chatapp/util/Linkify.kt index f0a8f93..453ff4f 100644 --- a/ChatApp/shared/src/main/kotlin/com/example/chatapp/util/Linkify.kt +++ b/ChatApp/shared/src/main/kotlin/com/example/chatapp/util/Linkify.kt @@ -24,7 +24,10 @@ import androidx.compose.ui.text.TextLinkStyles import androidx.compose.ui.text.buildAnnotatedString import androidx.compose.ui.text.style.TextDecoration -fun linkifyString(text: String, linkColor: Color? = null): AnnotatedString { +fun linkifyString( + text: String, + linkColor: Color? = null, +): AnnotatedString { val matcher = Patterns.WEB_URL.matcher(text) var lastIndex = 0 return buildAnnotatedString { @@ -35,22 +38,24 @@ fun linkifyString(text: String, linkColor: Color? = null): AnnotatedString { append(text.substring(lastIndex, start)) - val uriStr = if (url.startsWith("http://", ignoreCase = true) || url.startsWith("https://", ignoreCase = true)) { - url - } else { - "https://$url" - } - - val linkStyle = SpanStyle( - color = linkColor ?: Color.Unspecified, - textDecoration = TextDecoration.Underline - ) + val uriStr = + if (url.startsWith("http://", ignoreCase = true) || url.startsWith("https://", ignoreCase = true)) { + url + } else { + "https://$url" + } + + val linkStyle = + SpanStyle( + color = linkColor ?: Color.Unspecified, + textDecoration = TextDecoration.Underline, + ) pushLink( LinkAnnotation.Url( url = uriStr, - styles = TextLinkStyles(style = linkStyle) - ) + styles = TextLinkStyles(style = linkStyle), + ), ) append(url) pop() diff --git a/ChatApp/wear/build.gradle.kts b/ChatApp/wear/build.gradle.kts index fe02a1e..23bbd5b 100644 --- a/ChatApp/wear/build.gradle.kts +++ b/ChatApp/wear/build.gradle.kts @@ -67,7 +67,6 @@ dependencies { // App functions implementation(libs.androidx.appfunctions) ksp(libs.androidx.appfunctions.compiler) - } // AppFunctions ksp option diff --git a/ChatApp/wear/src/main/kotlin/com/example/chatapp/wear/uicomponents/WearChatScreen.kt b/ChatApp/wear/src/main/kotlin/com/example/chatapp/wear/uicomponents/WearChatScreen.kt index c710615..5b21082 100644 --- a/ChatApp/wear/src/main/kotlin/com/example/chatapp/wear/uicomponents/WearChatScreen.kt +++ b/ChatApp/wear/src/main/kotlin/com/example/chatapp/wear/uicomponents/WearChatScreen.kt @@ -70,9 +70,10 @@ fun WearChatScreen( title = { Text(text = if (message.isInbound) message.senderName ?: "Sender" else "Me") }, ) { val linkColor = MaterialTheme.colorScheme.primary - val annotatedText = remember(message.content, linkColor) { - linkifyString(text = message.content, linkColor = linkColor) - } + val annotatedText = + remember(message.content, linkColor) { + linkifyString(text = message.content, linkColor = linkColor) + } Text(text = annotatedText) } } diff --git a/agent/app/build.gradle.kts b/agent/app/build.gradle.kts index 75f0cac..ec6f854 100644 --- a/agent/app/build.gradle.kts +++ b/agent/app/build.gradle.kts @@ -61,13 +61,14 @@ android { dimension = "mode" buildConfigField("Boolean", "IS_RETAIL", "true") - val containsRetail = gradle.startParameter.taskNames.any { - it.contains("Retail", ignoreCase = true) - } + val containsRetail = + gradle.startParameter.taskNames.any { + it.contains("Retail", ignoreCase = true) + } val apiKey = project.findProperty("GEMINI_API_KEY") as? String ?: "" if (containsRetail && apiKey.isEmpty()) { throw GradleException( - "GEMINI_API_KEY project property is required for retail builds. Pass it using -PGEMINI_API_KEY=your_key" + "GEMINI_API_KEY project property is required for retail builds. Pass it using -PGEMINI_API_KEY=your_key", ) } buildConfigField("String", "GEMINI_API_KEY", "\"$apiKey\"") diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/BuiltInAppFunctions.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/BaseBuiltInAppFunctionService.kt similarity index 56% rename from agent/app/src/main/java/com/example/appfunctions/agent/data/BuiltInAppFunctions.kt rename to agent/app/src/main/java/com/example/appfunctions/agent/data/BaseBuiltInAppFunctionService.kt index 0d0f61d..912426e 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/BuiltInAppFunctions.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/BaseBuiltInAppFunctionService.kt @@ -25,7 +25,6 @@ import android.location.LocationManager import android.util.Base64 import androidx.annotation.RequiresApi import androidx.appfunctions.AppFunction -import androidx.appfunctions.AppFunctionContext import androidx.appfunctions.AppFunctionSerializable import androidx.appfunctions.AppFunctionService import androidx.appfunctions.AppFunctionServiceEntryPoint @@ -35,17 +34,17 @@ import androidx.datastore.preferences.core.stringPreferencesKey import com.example.appfunctions.agent.BuildConfig import com.example.appfunctions.agent.di.settingsDataStore import dagger.hilt.android.AndroidEntryPoint +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.withContext +import org.json.JSONArray +import org.json.JSONObject import java.io.File import java.net.HttpURLConnection import java.net.URL import java.util.UUID import kotlin.coroutines.resume import kotlin.coroutines.suspendCoroutine -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.flow.first -import kotlinx.coroutines.withContext -import org.json.JSONArray -import org.json.JSONObject /** Built-in AppFunctions for location and geocoding services. */ @RequiresApi(36) @@ -63,9 +62,7 @@ abstract class BaseBuiltInAppFunctionService : AppFunctionService() { * @return The latitude and longitude coordinates of the address, or null if geocoding fails. */ @AppFunction(isDescribedByKDoc = true) - suspend fun geocodeAddress( - address: String, - ): LatLng? { + suspend fun geocodeAddress(address: String): LatLng? { if (!Geocoder.isPresent()) { return null } @@ -83,7 +80,7 @@ abstract class BaseBuiltInAppFunctionService : AppFunctionService() { val location = addresses.firstOrNull() if (location != null) { continuation.resume( - LatLng(location.latitude, location.longitude) + LatLng(location.latitude, location.longitude), ) } else { continuation.resume(null) @@ -93,7 +90,7 @@ abstract class BaseBuiltInAppFunctionService : AppFunctionService() { override fun onError(errorMessage: String?) { continuation.resume(null) } - } + }, ) } } catch (e: Exception) { @@ -165,7 +162,7 @@ abstract class BaseBuiltInAppFunctionService : AppFunctionService() { /** The latitude coordinate. */ val latitude: Double, /** The longitude coordinate. */ - val longitude: Double + val longitude: Double, ) /** @@ -178,148 +175,149 @@ abstract class BaseBuiltInAppFunctionService : AppFunctionService() { @AppFunction(isDescribedByKDoc = true) suspend fun generateImage( prompt: String, - aspectRatio: String? = null - ): GeneratedImageResult = withContext(Dispatchers.IO) { - val context = this@BaseBuiltInAppFunctionService - val apiKey = - context.settingsDataStore.data - .first()[stringPreferencesKey("gemini_api_key")] - ?.takeIf { it.isNotBlank() } - ?: BuildConfig.GEMINI_API_KEY.takeIf { it.isNotBlank() } - if (apiKey.isNullOrBlank()) { - throw IllegalStateException( - "Gemini API key is not configured. Please set gemini_api_key in settings." - ) - } - - val requestJson = - JSONObject().apply { - put( - "contents", - JSONArray().apply { - put( - JSONObject().apply { - put( - "parts", - JSONArray().apply { - put(JSONObject().apply { put("text", prompt) }) - } - ) - } - ) - } + aspectRatio: String? = null, + ): GeneratedImageResult = + withContext(Dispatchers.IO) { + val context = this@BaseBuiltInAppFunctionService + val apiKey = + context.settingsDataStore.data + .first()[stringPreferencesKey("gemini_api_key")] + ?.takeIf { it.isNotBlank() } + ?: BuildConfig.GEMINI_API_KEY.takeIf { it.isNotBlank() } + if (apiKey.isNullOrBlank()) { + throw IllegalStateException( + "Gemini API key is not configured. Please set gemini_api_key in settings.", ) - put( - "generationConfig", - JSONObject().apply { - put("responseModalities", JSONArray().apply { put("IMAGE") }) - if (!aspectRatio.isNullOrBlank()) { + } + + val requestJson = + JSONObject().apply { + put( + "contents", + JSONArray().apply { put( - "imageConfig", JSONObject().apply { - put("aspectRatio", aspectRatio) - } + put( + "parts", + JSONArray().apply { + put(JSONObject().apply { put("text", prompt) }) + }, + ) + }, ) - } - } - ) - } + }, + ) + put( + "generationConfig", + JSONObject().apply { + put("responseModalities", JSONArray().apply { put("IMAGE") }) + if (!aspectRatio.isNullOrBlank()) { + put( + "imageConfig", + JSONObject().apply { + put("aspectRatio", aspectRatio) + }, + ) + } + }, + ) + } - val endpointUrl = - "https://generativelanguage.googleapis.com/v1beta/models/gemini-3.1-flash-image-preview:generateContent?key=$apiKey" - val url = URL(endpointUrl) - val connection = - (url.openConnection() as HttpURLConnection).apply { - requestMethod = "POST" - setRequestProperty("Content-Type", "application/json") - doOutput = true - connectTimeout = 30000 - readTimeout = 60000 - } + val endpointUrl = + "https://generativelanguage.googleapis.com/v1beta/models/gemini-3.1-flash-image-preview:generateContent?key=$apiKey" + val url = URL(endpointUrl) + val connection = + (url.openConnection() as HttpURLConnection).apply { + requestMethod = "POST" + setRequestProperty("Content-Type", "application/json") + doOutput = true + connectTimeout = 30000 + readTimeout = 60000 + } - try { - connection.outputStream.use { os -> - os.write(requestJson.toString().toByteArray(Charsets.UTF_8)) - } + try { + connection.outputStream.use { os -> + os.write(requestJson.toString().toByteArray(Charsets.UTF_8)) + } - val responseCode = connection.responseCode - if (responseCode != HttpURLConnection.HTTP_OK) { - val errorBody = - connection.errorStream?.bufferedReader()?.use { it.readText() } - ?: "HTTP $responseCode" - throw IllegalStateException( - "Image generation failed ($responseCode): $errorBody" - ) - } + val responseCode = connection.responseCode + if (responseCode != HttpURLConnection.HTTP_OK) { + val errorBody = + connection.errorStream?.bufferedReader()?.use { it.readText() } + ?: "HTTP $responseCode" + throw IllegalStateException( + "Image generation failed ($responseCode): $errorBody", + ) + } - val responseText = connection.inputStream.bufferedReader().use { it.readText() } - val responseJson = JSONObject(responseText) - val candidates = responseJson.optJSONArray("candidates") - if (candidates == null || candidates.length() == 0) { - throw IllegalStateException( - "No candidates returned from Gemini image generation API" - ) - } + val responseText = connection.inputStream.bufferedReader().use { it.readText() } + val responseJson = JSONObject(responseText) + val candidates = responseJson.optJSONArray("candidates") + if (candidates == null || candidates.length() == 0) { + throw IllegalStateException( + "No candidates returned from Gemini image generation API", + ) + } - val parts = - candidates - .getJSONObject(0) - .optJSONObject("content") - ?.optJSONArray("parts") - if (parts == null || parts.length() == 0) { - throw IllegalStateException( - "No parts returned in candidate content. Gemini response: $responseText" - ) - } + val parts = + candidates + .getJSONObject(0) + .optJSONObject("content") + ?.optJSONArray("parts") + if (parts == null || parts.length() == 0) { + throw IllegalStateException( + "No parts returned in candidate content. Gemini response: $responseText", + ) + } - var base64Data: String? = null - var mimeType = "image/png" - for (i in 0 until parts.length()) { - val part = parts.getJSONObject(i) - val inlineData = - part.optJSONObject("inlineData") - ?: part.optJSONObject("inline_data") - if (inlineData != null) { - base64Data = inlineData.optString("data") - val returnedMime = - inlineData - .optString("mimeType") - .takeIf { it.isNotBlank() } - ?: inlineData.optString("mime_type") - if (returnedMime.isNotBlank()) { - mimeType = returnedMime + var base64Data: String? = null + var mimeType = "image/png" + for (i in 0 until parts.length()) { + val part = parts.getJSONObject(i) + val inlineData = + part.optJSONObject("inlineData") + ?: part.optJSONObject("inline_data") + if (inlineData != null) { + base64Data = inlineData.optString("data") + val returnedMime = + inlineData + .optString("mimeType") + .takeIf { it.isNotBlank() } + ?: inlineData.optString("mime_type") + if (returnedMime.isNotBlank()) { + mimeType = returnedMime + } + break } - break } - } - if (base64Data.isNullOrBlank()) { - throw IllegalStateException( - "No inlineData image found in response parts. Gemini response: $responseText" - ) - } + if (base64Data.isNullOrBlank()) { + throw IllegalStateException( + "No inlineData image found in response parts. Gemini response: $responseText", + ) + } - val imageBytes = Base64.decode(base64Data, Base64.DEFAULT) - val extension = - if (mimeType.contains("jpeg") || mimeType.contains("jpg")) "jpg" else "png" - val cachedFile = - File( - context.cacheDir, - "generated_${UUID.randomUUID()}.$extension" - ) - cachedFile.writeBytes(imageBytes) + val imageBytes = Base64.decode(base64Data, Base64.DEFAULT) + val extension = + if (mimeType.contains("jpeg") || mimeType.contains("jpg")) "jpg" else "png" + val cachedFile = + File( + context.cacheDir, + "generated_${UUID.randomUUID()}.$extension", + ) + cachedFile.writeBytes(imageBytes) - val authority = "${context.packageName}.fileprovider" - val contentUri = FileProvider.getUriForFile(context, authority, cachedFile) - GeneratedImageResult( - imageUri = contentUri.toString(), - mimeType = mimeType, - prompt = prompt - ) - } finally { - connection.disconnect() + val authority = "${context.packageName}.fileprovider" + val contentUri = FileProvider.getUriForFile(context, authority, cachedFile) + GeneratedImageResult( + imageUri = contentUri.toString(), + mimeType = mimeType, + prompt = prompt, + ) + } finally { + connection.disconnect() + } } - } /** Represents the result of an image generation request. */ @AppFunctionSerializable @@ -329,6 +327,6 @@ abstract class BaseBuiltInAppFunctionService : AppFunctionService() { /** The MIME type of the generated image. */ val mimeType: String, /** The original prompt used to generate the image. */ - val prompt: String + val prompt: String, ) } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/FakeAppFunctions.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/BaseFakeAppFunctionService.kt similarity index 96% rename from agent/app/src/main/java/com/example/appfunctions/agent/data/FakeAppFunctions.kt rename to agent/app/src/main/java/com/example/appfunctions/agent/data/BaseFakeAppFunctionService.kt index 1f82650..2048ee9 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/FakeAppFunctions.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/BaseFakeAppFunctionService.kt @@ -37,9 +37,7 @@ abstract class BaseFakeAppFunctionService : AppFunctionService() { * @return The test response containing output data. */ @AppFunction(isDescribedByKDoc = true) - suspend fun fakeFunction( - params: FakeParams, - ): FakeResponse { + suspend fun fakeFunction(params: FakeParams): FakeResponse { return FakeResponse("success") } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt index e061216..62438db 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt @@ -29,9 +29,6 @@ import io.ktor.client.statement.HttpResponse import io.ktor.client.statement.bodyAsText import io.ktor.http.ContentType import io.ktor.http.contentType -import java.time.LocalDate -import javax.inject.Inject -import javax.inject.Singleton import kotlinx.serialization.json.Json import kotlinx.serialization.json.JsonArray import kotlinx.serialization.json.JsonElement @@ -42,220 +39,223 @@ import kotlinx.serialization.json.jsonArray import kotlinx.serialization.json.jsonObject import kotlinx.serialization.json.jsonPrimitive import kotlinx.serialization.json.put +import java.time.LocalDate +import javax.inject.Inject +import javax.inject.Singleton @Singleton class GeminiProviderImpl -@Inject -constructor( - private val httpClient: HttpClient, - private val toolConverter: GeminiToolConverter -) : LlmProvider { - override suspend fun generateResponse( - previousInteractionId: String?, - input: LlmInput, - tools: List, - apiKey: String, - modelName: String - ): LlmResponse { - val convertedTools = - tools.mapNotNull { tool -> - try { - buildJsonObject { - put(KEY_TYPE, JsonPrimitive(VALUE_FUNCTION)) - val functionSchema = toolConverter.convert(tool) - functionSchema.forEach { (key, value) -> put(key, value) } + @Inject + constructor( + private val httpClient: HttpClient, + private val toolConverter: GeminiToolConverter, + ) : LlmProvider { + override suspend fun generateResponse( + previousInteractionId: String?, + input: LlmInput, + tools: List, + apiKey: String, + modelName: String, + ): LlmResponse { + val convertedTools = + tools.mapNotNull { tool -> + try { + buildJsonObject { + put(KEY_TYPE, JsonPrimitive(VALUE_FUNCTION)) + val functionSchema = toolConverter.convert(tool) + functionSchema.forEach { (key, value) -> put(key, value) } + } + } catch (e: IllegalArgumentException) { + Log.e(TAG, "Failed to convert tool ${tool.id}: ${e.message}", e) + null } - } catch (e: IllegalArgumentException) { - Log.e(TAG, "Failed to convert tool ${tool.id}: ${e.message}", e) - null } - } - - val requestBody = - buildJsonObject { - put(KEY_MODEL, JsonPrimitive(modelName)) - put(KEY_SYSTEM_INSTRUCTION, JsonPrimitive(getSystemInstruction())) + val requestBody = + buildJsonObject { + put(KEY_MODEL, JsonPrimitive(modelName)) - when (input) { - is LlmInput.ToolResponse -> { - val inputElement = - kotlinx.serialization.json.buildJsonArray { - input.outputs.forEach { output -> - val matchingTool = tools.find { it.id == output.functionId } - val mappedName = - if (matchingTool != null) { - toolConverter.getToolName(matchingTool) - } else { - output.functionId - } + put(KEY_SYSTEM_INSTRUCTION, JsonPrimitive(getSystemInstruction())) - add( - buildJsonObject { - put("type", "function_result") - put("name", mappedName) - if (output.callId.isNotEmpty()) { - put("call_id", output.callId) + when (input) { + is LlmInput.ToolResponse -> { + val inputElement = + kotlinx.serialization.json.buildJsonArray { + input.outputs.forEach { output -> + val matchingTool = tools.find { it.id == output.functionId } + val mappedName = + if (matchingTool != null) { + toolConverter.getToolName(matchingTool) + } else { + output.functionId } - put("result", output.result) - } - ) + + add( + buildJsonObject { + put("type", "function_result") + put("name", mappedName) + if (output.callId.isNotEmpty()) { + put("call_id", output.callId) + } + put("result", output.result) + }, + ) + } } - } - put(KEY_INPUT, inputElement) + put(KEY_INPUT, inputElement) + } + is LlmInput.UserMessage -> { + put(KEY_INPUT, JsonPrimitive(input.text)) + } } - is LlmInput.UserMessage -> { - put(KEY_INPUT, JsonPrimitive(input.text)) + + if (convertedTools.isNotEmpty()) { + put(KEY_TOOLS, kotlinx.serialization.json.JsonArray(convertedTools)) } - } - if (convertedTools.isNotEmpty()) { - put(KEY_TOOLS, kotlinx.serialization.json.JsonArray(convertedTools)) + if (previousInteractionId != null) { + put(KEY_PREVIOUS_INTERACTION_ID, JsonPrimitive(previousInteractionId)) + } } - if (previousInteractionId != null) { - put(KEY_PREVIOUS_INTERACTION_ID, JsonPrimitive(previousInteractionId)) - } - } + Log.d(TAG, "Gemini Request Body: $requestBody") - Log.d(TAG, "Gemini Request Body: $requestBody") - - val response: HttpResponse = - try { - httpClient.post(GEMINI_INTERACTIONS_URL) { - contentType(ContentType.Application.Json) - header(HEADER_API_KEY, apiKey) - header(HEADER_API_REVISION, VALUE_API_REVISION) - setBody(requestBody) + val response: HttpResponse = + try { + httpClient.post(GEMINI_INTERACTIONS_URL) { + contentType(ContentType.Application.Json) + header(HEADER_API_KEY, apiKey) + header(HEADER_API_REVISION, VALUE_API_REVISION) + setBody(requestBody) + } + } catch (e: Exception) { + return LlmResponse.Error("Network error: ${e.message}") } - } catch (e: Exception) { - return LlmResponse.Error("Network error: ${e.message}") - } - val responseBodyText = response.bodyAsText() - Log.d(TAG, "Gemini Response Body: $responseBodyText") + val responseBodyText = response.bodyAsText() + Log.d(TAG, "Gemini Response Body: $responseBodyText") - if (response.status.value !in 200..299) { - return LlmResponse.Error("Gemini API error: ${response.status} - $responseBodyText") - } + if (response.status.value !in 200..299) { + return LlmResponse.Error("Gemini API error: ${response.status} - $responseBodyText") + } - val jsonResponse = Json.parseToJsonElement(responseBodyText).jsonObject + val jsonResponse = Json.parseToJsonElement(responseBodyText).jsonObject - val newInteractionId = - jsonResponse[KEY_ID]?.jsonPrimitive?.content - ?: return LlmResponse.Error("Missing interaction ID in response") - val steps = jsonResponse[KEY_STEPS]?.jsonArray + val newInteractionId = + jsonResponse[KEY_ID]?.jsonPrimitive?.content + ?: return LlmResponse.Error("Missing interaction ID in response") + val steps = jsonResponse[KEY_STEPS]?.jsonArray - val parts = mutableListOf() + val parts = mutableListOf() - if (steps != null) { - for (step in steps) { - val stepObj = step.jsonObject - val stepType = stepObj[KEY_TYPE]?.jsonPrimitive?.content + if (steps != null) { + for (step in steps) { + val stepObj = step.jsonObject + val stepType = stepObj[KEY_TYPE]?.jsonPrimitive?.content - if (stepType == VALUE_MODEL_OUTPUT) { - val content = stepObj[KEY_CONTENT]?.jsonArray - if (content != null) { - for (part in content) { - val partObj = part.jsonObject - val partType = partObj[KEY_TYPE]?.jsonPrimitive?.content - if (partType == VALUE_TEXT) { - val text = partObj[KEY_TEXT]?.jsonPrimitive?.content - if (text != null) { - parts.add(LlmResponsePart.Text(text)) + if (stepType == VALUE_MODEL_OUTPUT) { + val content = stepObj[KEY_CONTENT]?.jsonArray + if (content != null) { + for (part in content) { + val partObj = part.jsonObject + val partType = partObj[KEY_TYPE]?.jsonPrimitive?.content + if (partType == VALUE_TEXT) { + val text = partObj[KEY_TEXT]?.jsonPrimitive?.content + if (text != null) { + parts.add(LlmResponsePart.Text(text)) + } } } } - } - } else if (stepType == VALUE_FUNCTION_CALL) { - val name = - stepObj[KEY_NAME]?.jsonPrimitive?.content - ?: return LlmResponse.Error("Called function without a name") - val matchingTool = - tools.find { tool -> toolConverter.getToolName(tool) == name } - ?: return LlmResponse.Error("Called unknown function: $name") + } else if (stepType == VALUE_FUNCTION_CALL) { + val name = + stepObj[KEY_NAME]?.jsonPrimitive?.content + ?: return LlmResponse.Error("Called function without a name") + val matchingTool = + tools.find { tool -> toolConverter.getToolName(tool) == name } + ?: return LlmResponse.Error("Called unknown function: $name") - val packageName = matchingTool.packageName - val functionId = matchingTool.id + val packageName = matchingTool.packageName + val functionId = matchingTool.id - val args = stepObj[KEY_ARGUMENTS]?.jsonObject - val argumentsMap = mutableMapOf() - if (args != null) { - for ((key, value) in args) { - argumentsMap[key] = value.toPrimitive() + val args = stepObj[KEY_ARGUMENTS]?.jsonObject + val argumentsMap = mutableMapOf() + if (args != null) { + for ((key, value) in args) { + argumentsMap[key] = value.toPrimitive() + } } - } - val callId = - stepObj["id"]?.jsonPrimitive?.content - ?: return LlmResponse.Error( - "Function call missing call_id in response" - ) - parts.add( - LlmResponsePart.ToolCall( - packageName = packageName, - functionId = functionId, - arguments = argumentsMap, - callId = callId + val callId = + stepObj["id"]?.jsonPrimitive?.content + ?: return LlmResponse.Error( + "Function call missing call_id in response", + ) + parts.add( + LlmResponsePart.ToolCall( + packageName = packageName, + functionId = functionId, + arguments = argumentsMap, + callId = callId, + ), ) - ) - } else { - Log.d(TAG, "Unsupported step type: $stepType") + } else { + Log.d(TAG, "Unsupported step type: $stepType") + } } } - } - return LlmResponse.Success( - interactionId = newInteractionId, - parts = parts - ) - } + return LlmResponse.Success( + interactionId = newInteractionId, + parts = parts, + ) + } - private fun JsonElement.toPrimitive(): Any? { - return when (this) { - is JsonPrimitive -> { - if (this.isString) return this.content - return this.content.toBooleanStrictOrNull() - ?: this.content.toLongOrNull() - ?: this.content.toDoubleOrNull() + private fun JsonElement.toPrimitive(): Any? { + return when (this) { + is JsonPrimitive -> { + if (this.isString) return this.content + return this.content.toBooleanStrictOrNull() + ?: this.content.toLongOrNull() + ?: this.content.toDoubleOrNull() + } + is JsonObject -> this.mapValues { it.value.toPrimitive() } + is JsonArray -> this.map { it.toPrimitive() } } - is JsonObject -> this.mapValues { it.value.toPrimitive() } - is JsonArray -> this.map { it.toPrimitive() } } - } - private fun getSystemInstruction(): String { - val currentDate = LocalDate.now().toString() - return """ + private fun getSystemInstruction(): String { + val currentDate = LocalDate.now().toString() + return """ You are an AI assistant running on Android. Today's date is $currentDate. Always reply to the user using concise, friendly, natural human language. Never respond to the user with raw JSON or structured code blocks unless explicitly asked to write code. When a user asks you to generate an image, call the generateImage tool. After generateImage completes, confirm to the user in a natural sentence (for example: "I have generated that image for you."). When a user asks you to generate an image and use it in an app (for example, setting a chat wallpaper or attaching an image to a note), first call generateImage, then call the target app function passing the returned imageUri. - """.trimIndent() - } + """.trimIndent() + } - companion object { - private const val GEMINI_INTERACTIONS_URL = - "https://generativelanguage.googleapis.com/v1beta/interactions" - private const val TAG = "GeminiProvider" + companion object { + private const val GEMINI_INTERACTIONS_URL = + "https://generativelanguage.googleapis.com/v1beta/interactions" + private const val TAG = "GeminiProvider" - private const val KEY_MODEL = "model" - private const val KEY_INPUT = "input" - private const val KEY_TOOLS = "tools" - private const val KEY_TYPE = "type" - private const val VALUE_FUNCTION = "function" - private const val KEY_PREVIOUS_INTERACTION_ID = "previous_interaction_id" - private const val KEY_SYSTEM_INSTRUCTION = "system_instruction" - private const val HEADER_API_KEY = "x-goog-api-key" - private const val KEY_ID = "id" - private const val KEY_STEPS = "steps" - private const val KEY_CONTENT = "content" - private const val VALUE_MODEL_OUTPUT = "model_output" - private const val VALUE_TEXT = "text" - private const val KEY_TEXT = "text" - private const val VALUE_FUNCTION_CALL = "function_call" - private const val KEY_NAME = "name" - private const val KEY_ARGUMENTS = "arguments" - private const val HEADER_API_REVISION = "Api-Revision" - private const val VALUE_API_REVISION = "2026-05-20" + private const val KEY_MODEL = "model" + private const val KEY_INPUT = "input" + private const val KEY_TOOLS = "tools" + private const val KEY_TYPE = "type" + private const val VALUE_FUNCTION = "function" + private const val KEY_PREVIOUS_INTERACTION_ID = "previous_interaction_id" + private const val KEY_SYSTEM_INSTRUCTION = "system_instruction" + private const val HEADER_API_KEY = "x-goog-api-key" + private const val KEY_ID = "id" + private const val KEY_STEPS = "steps" + private const val KEY_CONTENT = "content" + private const val VALUE_MODEL_OUTPUT = "model_output" + private const val VALUE_TEXT = "text" + private const val KEY_TEXT = "text" + private const val VALUE_FUNCTION_CALL = "function_call" + private const val KEY_NAME = "name" + private const val KEY_ARGUMENTS = "arguments" + private const val HEADER_API_REVISION = "Api-Revision" + private const val VALUE_API_REVISION = "2026-05-20" + } } -} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/db/entities/MessageEntity.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/db/entities/MessageEntity.kt index 7144b03..32022ba 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/db/entities/MessageEntity.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/db/entities/MessageEntity.kt @@ -26,15 +26,15 @@ import kotlinx.serialization.json.Json @Entity( tableName = "messages", foreignKeys = - [ - ForeignKey( - entity = ThreadEntity::class, - parentColumns = ["threadId"], - childColumns = ["threadId"], - onDelete = ForeignKey.CASCADE - ) - ], - indices = [Index("threadId")] + [ + ForeignKey( + entity = ThreadEntity::class, + parentColumns = ["threadId"], + childColumns = ["threadId"], + onDelete = ForeignKey.CASCADE, + ), + ], + indices = [Index("threadId")], ) data class MessageEntity( @PrimaryKey val messageId: String, @@ -49,24 +49,24 @@ data class MessageEntity( */ val pendingIntentId: String? = null, val targetPackageName: String? = null, - val attachments: List = emptyList() + val attachments: List = emptyList(), ) @Serializable data class MessageAttachment( val uri: String, - val mimeType: String + val mimeType: String, ) class MessageAttachmentConverter { - private val dbJson = Json { - ignoreUnknownKeys = true - encodeDefaults = true - } + private val dbJson = + Json { + ignoreUnknownKeys = true + encodeDefaults = true + } @androidx.room.TypeConverter - fun fromAttachments(attachments: List): String = - dbJson.encodeToString(attachments) + fun fromAttachments(attachments: List): String = dbJson.encodeToString(attachments) @androidx.room.TypeConverter fun toAttachments(jsonString: String?): List = @@ -80,11 +80,11 @@ class MessageAttachmentConverter { enum class MessageRole { USER, - ASSISTANT + ASSISTANT, } enum class MessageProcessingStatus { PENDING_AGENT_RESPONSE, PROCESSED, - FAILED + FAILED, } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt b/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt index bdc1410..3757f9d 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/di/DataModule.kt @@ -39,8 +39,8 @@ import io.ktor.client.engine.okhttp.OkHttp import io.ktor.client.plugins.HttpTimeout import io.ktor.client.plugins.contentnegotiation.ContentNegotiation import io.ktor.serialization.kotlinx.json.json -import javax.inject.Singleton import kotlinx.serialization.json.Json +import javax.inject.Singleton val Context.settingsDataStore: DataStore by preferencesDataStore(name = "settings") @@ -58,30 +58,32 @@ abstract class DataModule { @Binds @Singleton - abstract fun bindPendingIntentRepository( - impl: InMemoryPendingIntentRepository - ): PendingIntentRepository + abstract fun bindPendingIntentRepository(impl: InMemoryPendingIntentRepository): PendingIntentRepository @Binds @Singleton abstract fun bindLlmProviderFactory( - impl: com.example.appfunctions.agent.data.LlmProviderFactoryImpl + impl: com.example.appfunctions.agent.data.LlmProviderFactoryImpl, ): com.example.appfunctions.agent.domain.LlmProviderFactory companion object { @Provides @Singleton - fun provideDataStore(@ApplicationContext context: Context): DataStore { + fun provideDataStore( + @ApplicationContext context: Context, + ): DataStore { return context.settingsDataStore } @Provides @Singleton - fun provideAppDatabase(@ApplicationContext context: Context): AppDatabase { + fun provideAppDatabase( + @ApplicationContext context: Context, + ): AppDatabase { return Room.databaseBuilder( context, AppDatabase::class.java, - "app_database" + "app_database", ) .fallbackToDestructiveMigration() .build() @@ -103,7 +105,7 @@ abstract class DataModule { ignoreUnknownKeys = true prettyPrint = true isLenient = true - } + }, ) } install(HttpTimeout) { socketTimeoutMillis = 30000 } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt index ec5163d..df9c8d3 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt @@ -20,7 +20,6 @@ import android.content.Context import android.content.Intent import android.net.Uri import android.util.Log -import androidx.appfunctions.AppFunctionException import androidx.appfunctions.metadata.AppFunctionMetadata import com.example.appfunctions.agent.data.LlmProviderName import com.example.appfunctions.agent.data.SettingsRepository @@ -43,9 +42,6 @@ import com.example.appfunctions.agent.domain.chat.UpdateThreadParams import com.example.appfunctions.agent.domain.chat.UpdateThreadUseCase import com.example.appfunctions.agent.domain.pendingintent.SavePendingIntentUseCase import dagger.hilt.android.qualifiers.ApplicationContext -import java.util.UUID -import javax.inject.Inject -import javax.inject.Singleton import kotlinx.coroutines.CancellationException import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.coroutineScope @@ -58,412 +54,419 @@ import kotlinx.coroutines.flow.first import kotlinx.coroutines.flow.stateIn import kotlinx.coroutines.withContext import org.json.JSONObject +import java.util.UUID +import javax.inject.Inject +import javax.inject.Singleton /** Orchestrates the interaction between the LLM, AppFunctions, and the chat repository. */ @Singleton class AgentOrchestrator -@Inject -constructor( - @ApplicationContext private val context: Context, - private val manageThreadsUseCase: ManageThreadsUseCase, - private val observePendingMessagesUseCase: ObservePendingMessagesUseCase, - private val sendMessageUseCase: SendMessageUseCase, - private val updateMessageUseCase: UpdateMessageUseCase, - private val updateThreadUseCase: UpdateThreadUseCase, - private val llmProviderFactory: LlmProviderFactory, - private val settingsRepository: SettingsRepository, - private val getAppFunctionsUseCase: GetAppFunctionsUseCase, - private val convertInputToAppFunctionDataUseCase: ConvertInputToAppFunctionDataUseCase, - private val executeAppFunctionUseCase: ExecuteAppFunctionUseCase, - private val savePendingIntentUseCase: SavePendingIntentUseCase -) { - private val _status = MutableStateFlow(AgentStatus.Idle) - - /** The current status of the agent. */ - val status: StateFlow = _status.asStateFlow() - - /** - * Observes messages for a specific thread and processes any pending agent responses. This - * method suspends and collects messages until cancelled. - */ - suspend fun observeAndProcessMessages(threadId: String) = coroutineScope { - val threadStateFlow = - manageThreadsUseCase.getThread( - threadId - ).stateIn(this, SharingStarted.Eagerly, null) - - observePendingMessagesUseCase(threadId).collect { message -> - if (message != null) { - val thread = threadStateFlow.filterNotNull().first() - processMessage(message, thread) + @Inject + constructor( + @ApplicationContext private val context: Context, + private val manageThreadsUseCase: ManageThreadsUseCase, + private val observePendingMessagesUseCase: ObservePendingMessagesUseCase, + private val sendMessageUseCase: SendMessageUseCase, + private val updateMessageUseCase: UpdateMessageUseCase, + private val updateThreadUseCase: UpdateThreadUseCase, + private val llmProviderFactory: LlmProviderFactory, + private val settingsRepository: SettingsRepository, + private val getAppFunctionsUseCase: GetAppFunctionsUseCase, + private val convertInputToAppFunctionDataUseCase: ConvertInputToAppFunctionDataUseCase, + private val executeAppFunctionUseCase: ExecuteAppFunctionUseCase, + private val savePendingIntentUseCase: SavePendingIntentUseCase, + ) { + private val _status = MutableStateFlow(AgentStatus.Idle) + + /** The current status of the agent. */ + val status: StateFlow = _status.asStateFlow() + + /** + * Observes messages for a specific thread and processes any pending agent responses. This + * method suspends and collects messages until cancelled. + */ + suspend fun observeAndProcessMessages(threadId: String) = + coroutineScope { + val threadStateFlow = + manageThreadsUseCase.getThread( + threadId, + ).stateIn(this, SharingStarted.Eagerly, null) + + observePendingMessagesUseCase(threadId).collect { message -> + if (message != null) { + val thread = threadStateFlow.filterNotNull().first() + processMessage(message, thread) + } + } } - } - } - private suspend fun processMessage(message: MessageEntity, thread: ThreadEntity) { - _status.value = AgentStatus.Thinking + private suspend fun processMessage( + message: MessageEntity, + thread: ThreadEntity, + ) { + _status.value = AgentStatus.Thinking + + try { + val provider = thread.llmModel.providerName + val apiKey = getApiKey(provider) + if (apiKey == null) { + completeMessageWithError( + message.messageId, + message.threadId, + "API key is missing for ${provider.name}", + ) + _status.value = AgentStatus.Idle + return + } + + val disconnectedApps = settingsRepository.disconnectedApps.first() + val allTools = getAppFunctionsUseCase().first().values.flatten() - try { - val provider = thread.llmModel.providerName - val apiKey = getApiKey(provider) - if (apiKey == null) { + val targetPackageName = message.targetPackageName + val queryText = message.textContent + + val tools = filterTools(allTools, disconnectedApps, targetPackageName) + + runInteractionLoop( + message = message, + thread = thread, + apiKey = apiKey, + tools = tools, + initialInput = queryText, + ) + + _status.value = AgentStatus.Idle + } catch (e: Exception) { + Log.e("AgentOrchestrator", "Error processing message", e) completeMessageWithError( message.messageId, message.threadId, - "API key is missing for ${provider.name}" + e.message ?: "Unknown error occurred", ) _status.value = AgentStatus.Idle - return } - - val disconnectedApps = settingsRepository.disconnectedApps.first() - val allTools = getAppFunctionsUseCase().first().values.flatten() - - val targetPackageName = message.targetPackageName - val queryText = message.textContent - - val tools = filterTools(allTools, disconnectedApps, targetPackageName) - - runInteractionLoop( - message = message, - thread = thread, - apiKey = apiKey, - tools = tools, - initialInput = queryText - ) - - _status.value = AgentStatus.Idle - } catch (e: Exception) { - Log.e("AgentOrchestrator", "Error processing message", e) - completeMessageWithError( - message.messageId, - message.threadId, - e.message ?: "Unknown error occurred" - ) - _status.value = AgentStatus.Idle } - } - private fun filterTools( - allTools: List, - disconnectedApps: Set, - targetPackageName: String? - ): List { - return allTools.filter { metadata -> - metadata.isEnabled && - metadata.packageName !in disconnectedApps && - (targetPackageName == null || metadata.packageName == targetPackageName) + private fun filterTools( + allTools: List, + disconnectedApps: Set, + targetPackageName: String?, + ): List { + return allTools.filter { metadata -> + metadata.isEnabled && + metadata.packageName !in disconnectedApps && + (targetPackageName == null || metadata.packageName == targetPackageName) + } } - } - private suspend fun runInteractionLoop( - message: MessageEntity, - thread: ThreadEntity, - apiKey: String, - tools: List, - initialInput: String - ) { - val provider = thread.llmModel.providerName - val modelName = thread.llmModel.modelName - val llmProvider = llmProviderFactory.getProvider(provider) - - var previousInteractionId = thread.latestInteractionId - var currentToolOutputs = emptyList() - var continueLoop = true - var currentInput = initialInput - val capturedAttachments = mutableListOf() - - while (continueLoop) { - val llmInput = prepareLlmInput(currentToolOutputs, currentInput) - - currentToolOutputs = emptyList() - val response = - llmProvider.generateResponse( - previousInteractionId = previousInteractionId, - input = llmInput, - tools = tools, - apiKey = apiKey, - modelName = modelName - ) + private suspend fun runInteractionLoop( + message: MessageEntity, + thread: ThreadEntity, + apiKey: String, + tools: List, + initialInput: String, + ) { + val provider = thread.llmModel.providerName + val modelName = thread.llmModel.modelName + val llmProvider = llmProviderFactory.getProvider(provider) + + var previousInteractionId = thread.latestInteractionId + var currentToolOutputs = emptyList() + var continueLoop = true + var currentInput = initialInput + val capturedAttachments = mutableListOf() + + while (continueLoop) { + val llmInput = prepareLlmInput(currentToolOutputs, currentInput) + + currentToolOutputs = emptyList() + val response = + llmProvider.generateResponse( + previousInteractionId = previousInteractionId, + input = llmInput, + tools = tools, + apiKey = apiKey, + modelName = modelName, + ) - when ( - val handleResult = - handleLlmResponse(response, message, tools, capturedAttachments) - ) { - is HandleResult.Continue -> { - currentToolOutputs = handleResult.toolOutputs - previousInteractionId = handleResult.interactionId - } + when ( + val handleResult = + handleLlmResponse(response, message, tools, capturedAttachments) + ) { + is HandleResult.Continue -> { + currentToolOutputs = handleResult.toolOutputs + previousInteractionId = handleResult.interactionId + } - is HandleResult.Stop -> { - continueLoop = false + is HandleResult.Stop -> { + continueLoop = false + } } } } - } - private suspend fun getApiKey(provider: LlmProviderName): String? { - return when (provider) { - LlmProviderName.GEMINI -> settingsRepository.geminiApiKey.first() + private suspend fun getApiKey(provider: LlmProviderName): String? { + return when (provider) { + LlmProviderName.GEMINI -> settingsRepository.geminiApiKey.first() + } } - } - private fun prepareLlmInput( - currentToolOutputs: List, - currentInput: String - ): LlmInput { - return if (currentToolOutputs.isNotEmpty()) { - LlmInput.ToolResponse(currentToolOutputs) - } else { - LlmInput.UserMessage(currentInput) + private fun prepareLlmInput( + currentToolOutputs: List, + currentInput: String, + ): LlmInput { + return if (currentToolOutputs.isNotEmpty()) { + LlmInput.ToolResponse(currentToolOutputs) + } else { + LlmInput.UserMessage(currentInput) + } } - } - private sealed class HandleResult { - data class Continue(val toolOutputs: List, val interactionId: String) : - HandleResult() + private sealed class HandleResult { + data class Continue(val toolOutputs: List, val interactionId: String) : + HandleResult() - object Stop : HandleResult() - } + object Stop : HandleResult() + } - private sealed class ExecuteToolCallsResult { - data class Success(val toolOutputs: List) : ExecuteToolCallsResult() + private sealed class ExecuteToolCallsResult { + data class Success(val toolOutputs: List) : ExecuteToolCallsResult() - data class PendingIntentAction( - val pendingIntentId: String, - val pendingIntent: PendingIntent, - ) : ExecuteToolCallsResult() + data class PendingIntentAction( + val pendingIntentId: String, + val pendingIntent: PendingIntent, + ) : ExecuteToolCallsResult() - object Error : ExecuteToolCallsResult() - } + object Error : ExecuteToolCallsResult() + } - private suspend fun handleLlmResponse( - response: LlmResponse, - message: MessageEntity, - tools: List, - capturedAttachments: MutableList - ): HandleResult { - return when (response) { - is LlmResponse.Success -> { - updateThreadUseCase( - message.threadId, - UpdateThreadParams(interactionId = response.interactionId) - ) - updateMessageUseCase( - message.messageId, - UpdateMessageParams(status = MessageProcessingStatus.PROCESSED) - ) + private suspend fun handleLlmResponse( + response: LlmResponse, + message: MessageEntity, + tools: List, + capturedAttachments: MutableList, + ): HandleResult { + return when (response) { + is LlmResponse.Success -> { + updateThreadUseCase( + message.threadId, + UpdateThreadParams(interactionId = response.interactionId), + ) + updateMessageUseCase( + message.messageId, + UpdateMessageParams(status = MessageProcessingStatus.PROCESSED), + ) - val toolCalls = response.parts.filterIsInstance() - val textParts = response.parts.filterIsInstance() - - val textContent = textParts.joinToString("\n") { it.text } - - if (toolCalls.isNotEmpty()) { - when (val toolResult = executeToolCalls(toolCalls, tools, message)) { - is ExecuteToolCallsResult.Success -> { - for (output in toolResult.toolOutputs) { - runCatching { - val json = JSONObject(output.result) - val uri = json.optString("imageUri") - if (uri.isNotBlank()) { - val mimeType = - json.optString("mimeType") - .takeIf { it.isNotBlank() } - ?: "image/png" - capturedAttachments.add( - MessageAttachment(uri = uri, mimeType = mimeType) - ) + val toolCalls = response.parts.filterIsInstance() + val textParts = response.parts.filterIsInstance() + + val textContent = textParts.joinToString("\n") { it.text } + + if (toolCalls.isNotEmpty()) { + when (val toolResult = executeToolCalls(toolCalls, tools, message)) { + is ExecuteToolCallsResult.Success -> { + for (output in toolResult.toolOutputs) { + runCatching { + val json = JSONObject(output.result) + val uri = json.optString("imageUri") + if (uri.isNotBlank()) { + val mimeType = + json.optString("mimeType") + .takeIf { it.isNotBlank() } + ?: "image/png" + capturedAttachments.add( + MessageAttachment(uri = uri, mimeType = mimeType), + ) + } } } + if (textContent.isNotEmpty()) { + sendMessageUseCase( + threadId = message.threadId, + role = MessageRole.ASSISTANT, + textContent = textContent, + processingStatus = MessageProcessingStatus.PROCESSED, + ) + } + HandleResult.Continue( + toolResult.toolOutputs, + response.interactionId, + ) } - if (textContent.isNotEmpty()) { + + is ExecuteToolCallsResult.PendingIntentAction -> { + savePendingIntentUseCase( + toolResult.pendingIntentId, + toolResult.pendingIntent, + ) sendMessageUseCase( threadId = message.threadId, role = MessageRole.ASSISTANT, textContent = textContent, - processingStatus = MessageProcessingStatus.PROCESSED + processingStatus = MessageProcessingStatus.PROCESSED, + pendingIntentId = toolResult.pendingIntentId, ) + HandleResult.Stop } - HandleResult.Continue( - toolResult.toolOutputs, - response.interactionId - ) - } - is ExecuteToolCallsResult.PendingIntentAction -> { - savePendingIntentUseCase( - toolResult.pendingIntentId, - toolResult.pendingIntent - ) + is ExecuteToolCallsResult.Error -> { + HandleResult.Stop + } + } + } else { + if (textContent.isNotEmpty() || capturedAttachments.isNotEmpty()) { sendMessageUseCase( threadId = message.threadId, role = MessageRole.ASSISTANT, textContent = textContent, processingStatus = MessageProcessingStatus.PROCESSED, - pendingIntentId = toolResult.pendingIntentId + attachments = capturedAttachments, ) - HandleResult.Stop - } - - is ExecuteToolCallsResult.Error -> { - HandleResult.Stop } + HandleResult.Stop } - } else { - if (textContent.isNotEmpty() || capturedAttachments.isNotEmpty()) { - sendMessageUseCase( - threadId = message.threadId, - role = MessageRole.ASSISTANT, - textContent = textContent, - processingStatus = MessageProcessingStatus.PROCESSED, - attachments = capturedAttachments - ) - } - HandleResult.Stop } - } - - is LlmResponse.Error -> { - Log.e("AgentOrchestrator", "LLM Error: ${response.errorMessage}") - completeMessageWithError( - message.messageId, - message.threadId, - response.errorMessage - ) - _status.value = AgentStatus.Idle - HandleResult.Stop - } - } - } - private suspend fun executeToolCalls( - toolCalls: List, - tools: List, - message: MessageEntity - ): ExecuteToolCallsResult { - val results = mutableListOf() - for (toolCall in toolCalls) { - _status.value = AgentStatus.InvokingTool(toolCall.functionId, toolCall.packageName) - - val matchingTool = - tools.find { - it.packageName == toolCall.packageName && it.id == toolCall.functionId + is LlmResponse.Error -> { + Log.e("AgentOrchestrator", "LLM Error: ${response.errorMessage}") + completeMessageWithError( + message.messageId, + message.threadId, + response.errorMessage, + ) + _status.value = AgentStatus.Idle + HandleResult.Stop } - - if (matchingTool == null) { - completeMessageWithError( - message.messageId, - message.threadId, - "Tool not found: ${toolCall.functionId}" - ) - return ExecuteToolCallsResult.Error } + } - val convertedInputs = toolCall.arguments.filterValues { it != null } as Map - - for (value in convertedInputs.values) { - if (value is String && value.startsWith("content://")) { - runCatching { - val uri = Uri.parse(value) - context.grantUriPermission( - toolCall.packageName, - uri, - Intent.FLAG_GRANT_READ_URI_PERMISSION - ) + private suspend fun executeToolCalls( + toolCalls: List, + tools: List, + message: MessageEntity, + ): ExecuteToolCallsResult { + val results = mutableListOf() + for (toolCall in toolCalls) { + _status.value = AgentStatus.InvokingTool(toolCall.functionId, toolCall.packageName) + + val matchingTool = + tools.find { + it.packageName == toolCall.packageName && it.id == toolCall.functionId } - } - } - val appFunctionDataResult = - withContext(Dispatchers.Default) { - convertInputToAppFunctionDataUseCase( - parameters = matchingTool.parameters, - components = matchingTool.components, - inputs = convertedInputs + if (matchingTool == null) { + completeMessageWithError( + message.messageId, + message.threadId, + "Tool not found: ${toolCall.functionId}", ) + return ExecuteToolCallsResult.Error } - if (appFunctionDataResult.isFailure) { - completeMessageWithError( - message.messageId, - message.threadId, - "Failed to convert arguments for ${toolCall.functionId}\n ${appFunctionDataResult.exceptionOrNull()}" - ) - return ExecuteToolCallsResult.Error - } + val convertedInputs = toolCall.arguments.filterValues { it != null } as Map - val executionResult = - executeAppFunctionUseCase( - function = matchingTool, - parameters = appFunctionDataResult.getOrThrow(), - threadId = message.threadId - ) + for (value in convertedInputs.values) { + if (value is String && value.startsWith("content://")) { + runCatching { + val uri = Uri.parse(value) + context.grantUriPermission( + toolCall.packageName, + uri, + Intent.FLAG_GRANT_READ_URI_PERMISSION, + ) + } + } + } - when (executionResult) { - is ExecuteAppFunctionResult.Data -> { - results.add( - ToolOutput( - functionId = toolCall.functionId, - callId = toolCall.callId, - result = executionResult.formattedJson + val appFunctionDataResult = + withContext(Dispatchers.Default) { + convertInputToAppFunctionDataUseCase( + parameters = matchingTool.parameters, + components = matchingTool.components, + inputs = convertedInputs, ) + } + + if (appFunctionDataResult.isFailure) { + completeMessageWithError( + message.messageId, + message.threadId, + "Failed to convert arguments for ${toolCall.functionId}\n ${appFunctionDataResult.exceptionOrNull()}", ) + return ExecuteToolCallsResult.Error } - is ExecuteAppFunctionResult.PendingIntentAction -> { - val pendingIntentId = java.util.UUID.randomUUID().toString() - return ExecuteToolCallsResult.PendingIntentAction( - pendingIntentId, - executionResult.pendingIntent + val executionResult = + executeAppFunctionUseCase( + function = matchingTool, + parameters = appFunctionDataResult.getOrThrow(), + threadId = message.threadId, ) - } - is ExecuteAppFunctionResult.Error -> { - val exception = executionResult.exception - if (exception is CancellationException) { - throw exception - } - val appFunctionException = - AppFunctionExceptionFormatter.getAppFunctionException(exception) - if (appFunctionException != null) { + when (executionResult) { + is ExecuteAppFunctionResult.Data -> { results.add( ToolOutput( functionId = toolCall.functionId, callId = toolCall.callId, - result = - AppFunctionExceptionFormatter.format( - appFunctionException, - toolCall.functionId, - ), + result = executionResult.formattedJson, ), ) - } else { - throw IllegalStateException( - "Tool execution failed for ${toolCall.functionId}: ${exception.message}", - exception, + } + + is ExecuteAppFunctionResult.PendingIntentAction -> { + val pendingIntentId = java.util.UUID.randomUUID().toString() + return ExecuteToolCallsResult.PendingIntentAction( + pendingIntentId, + executionResult.pendingIntent, ) } + + is ExecuteAppFunctionResult.Error -> { + val exception = executionResult.exception + if (exception is CancellationException) { + throw exception + } + val appFunctionException = + AppFunctionExceptionFormatter.getAppFunctionException(exception) + if (appFunctionException != null) { + results.add( + ToolOutput( + functionId = toolCall.functionId, + callId = toolCall.callId, + result = + AppFunctionExceptionFormatter.format( + appFunctionException, + toolCall.functionId, + ), + ), + ) + } else { + throw IllegalStateException( + "Tool execution failed for ${toolCall.functionId}: ${exception.message}", + exception, + ) + } + } } } + return ExecuteToolCallsResult.Success(results) } - return ExecuteToolCallsResult.Success(results) - } - private suspend fun completeMessageWithError( - messageId: String, - threadId: String, - reason: String - ) { - updateMessageUseCase( - messageId, - UpdateMessageParams(status = MessageProcessingStatus.PROCESSED) - ) - sendMessageUseCase( - threadId = threadId, - role = MessageRole.ASSISTANT, - textContent = reason, - processingStatus = MessageProcessingStatus.FAILED - ) + private suspend fun completeMessageWithError( + messageId: String, + threadId: String, + reason: String, + ) { + updateMessageUseCase( + messageId, + UpdateMessageParams(status = MessageProcessingStatus.PROCESSED), + ) + sendMessageUseCase( + threadId = threadId, + role = MessageRole.ASSISTANT, + textContent = reason, + processingStatus = MessageProcessingStatus.FAILED, + ) + } } -} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/AppFunctionExceptionFormatter.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/AppFunctionExceptionFormatter.kt index c3b381a..fc42248 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/AppFunctionExceptionFormatter.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/AppFunctionExceptionFormatter.kt @@ -22,7 +22,10 @@ object AppFunctionExceptionFormatter { /** * Formats the given [exception] including its class name and message. */ - fun format(exception: AppFunctionException, functionId: String? = null): String { + fun format( + exception: AppFunctionException, + functionId: String? = null, + ): String { val className = exception.javaClass.simpleName val message = exception.errorMessage ?: exception.message ?: "No error message provided" val prefix = if (functionId != null) "Tool execution failed for $functionId: " else "" diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/chat/SendMessageUseCase.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/chat/SendMessageUseCase.kt index ef11560..4f9ac22 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/chat/SendMessageUseCase.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/chat/SendMessageUseCase.kt @@ -25,39 +25,39 @@ import javax.inject.Inject /** Use case to send a message in a chat thread. */ class SendMessageUseCase -@Inject -constructor( - private val chatRepository: ChatRepository -) { - /** - * Executes the use case. - * - * @param threadId The ID of the thread to send the message to. - * @param role The role of the sender (USER or ASSISTANT). - * @param textContent The content of the message. - * @param processingStatus The processing status of the message. - */ - suspend operator fun invoke( - threadId: String, - role: MessageRole, - textContent: String, - processingStatus: MessageProcessingStatus, - pendingIntentId: String? = null, - targetPackageName: String? = null, - attachments: List = emptyList() + @Inject + constructor( + private val chatRepository: ChatRepository, ) { - val message = - MessageEntity( - messageId = UUID.randomUUID().toString(), - threadId = threadId, - role = role, - textContent = textContent, - timestamp = System.currentTimeMillis(), - processingStatus = processingStatus, - pendingIntentId = pendingIntentId, - targetPackageName = targetPackageName, - attachments = attachments - ) - chatRepository.sendMessage(message) + /** + * Executes the use case. + * + * @param threadId The ID of the thread to send the message to. + * @param role The role of the sender (USER or ASSISTANT). + * @param textContent The content of the message. + * @param processingStatus The processing status of the message. + */ + suspend operator fun invoke( + threadId: String, + role: MessageRole, + textContent: String, + processingStatus: MessageProcessingStatus, + pendingIntentId: String? = null, + targetPackageName: String? = null, + attachments: List = emptyList(), + ) { + val message = + MessageEntity( + messageId = UUID.randomUUID().toString(), + threadId = threadId, + role = role, + textContent = textContent, + timestamp = System.currentTimeMillis(), + processingStatus = processingStatus, + pendingIntentId = pendingIntentId, + targetPackageName = targetPackageName, + attachments = attachments, + ) + chatRepository.sendMessage(message) + } } -} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt index 6c9758a..193e37c 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/agentdemo/AgentDemoScreen.kt @@ -145,7 +145,7 @@ fun AgentDemoScreen(viewModel: AgentDemoViewModel = hiltViewModel()) { fun AgentDemoContent( uiState: AgentUiState, onEvent: (AgentUiEvent) -> Unit, - initialSidePanelVisible: Boolean = false + initialSidePanelVisible: Boolean = false, ) { val context = LocalContext.current val packageManager = context.packageManager @@ -174,7 +174,7 @@ fun AgentDemoContent( drawerState = drawerState, scope = scope, packageManager = packageManager, - initialSidePanelVisible = initialSidePanelVisible + initialSidePanelVisible = initialSidePanelVisible, ) } } @@ -190,7 +190,7 @@ fun AgentDemoContent( drawerState = drawerState, drawerContent = { ModalDrawerSheet( - drawerContainerColor = MaterialTheme.colorScheme.surface + drawerContainerColor = MaterialTheme.colorScheme.surface, ) { ChatHistorySidePanel( threads = threads, @@ -198,10 +198,10 @@ fun AgentDemoContent( onEvent = { event -> onEvent(event) scope.launch { drawerState.close() } - } + }, ) } - } + }, ) { content() } @@ -224,7 +224,7 @@ fun AgentDemoLoadedScreen( drawerState: DrawerState, scope: CoroutineScope, packageManager: PackageManager, - initialSidePanelVisible: Boolean = false + initialSidePanelVisible: Boolean = false, ) { var messageText by remember { mutableStateOf(TextFieldValue("")) } var isSidePanelVisible by remember { mutableStateOf(initialSidePanelVisible) } @@ -243,13 +243,13 @@ fun AgentDemoLoadedScreen( topBar = { Row( modifier = Modifier.padding(horizontal = 8.dp, vertical = 16.dp), - verticalAlignment = Alignment.CenterVertically + verticalAlignment = Alignment.CenterVertically, ) { ModelDropdown( modifier = - Modifier - .weight(1f) - .padding(horizontal = 8.dp), + Modifier + .weight(1f) + .padding(horizontal = 8.dp), currentThread = uiState.currentThread, onModelSelected = { onEvent(AgentUiEvent.OnModelSelected(it)) }, onMenuClick = { @@ -258,39 +258,39 @@ fun AgentDemoLoadedScreen( } else { scope.launch { drawerState.open() } } - } + }, ) IconButton( onClick = { onEvent(AgentUiEvent.OnCreateThread(uiState.currentThread.llmModel)) }, - modifier = Modifier.padding(horizontal = 8.dp) + modifier = Modifier.padding(horizontal = 8.dp), ) { Icon(imageVector = Icons.Default.Add, contentDescription = "Create Thread") } } - } + }, ) { paddingValues -> Row( modifier = - Modifier - .fillMaxSize() - .imePadding() - .padding( - top = paddingValues.calculateTopPadding() - ) + Modifier + .fillMaxSize() + .imePadding() + .padding( + top = paddingValues.calculateTopPadding(), + ), ) { // Side Panel (only for wide screens) if (isWideScreen) { AnimatedVisibility( visible = isSidePanelVisible, enter = slideInHorizontally() + expandHorizontally(), - exit = slideOutHorizontally() + shrinkHorizontally() + exit = slideOutHorizontally() + shrinkHorizontally(), ) { ChatHistorySidePanel( threads = uiState.threads, currentThread = uiState.currentThread, - onEvent = onEvent + onEvent = onEvent, ) } } @@ -298,20 +298,20 @@ fun AgentDemoLoadedScreen( // Main Chat Area Column( modifier = - Modifier - .weight(1f) - .fillMaxHeight() - .padding(start = 16.dp, end = 16.dp), - verticalArrangement = Arrangement.SpaceBetween + Modifier + .weight(1f) + .fillMaxHeight() + .padding(start = 16.dp, end = 16.dp), + verticalArrangement = Arrangement.SpaceBetween, ) { // Messages List LazyColumn( modifier = - Modifier - .weight(1f) - .fillMaxWidth() - .clip(RoundedCornerShape(16.dp)), - reverseLayout = true + Modifier + .weight(1f) + .fillMaxWidth() + .clip(RoundedCornerShape(16.dp)), + reverseLayout = true, ) { // Status item at the bottom (above input) if not // idle @@ -319,21 +319,21 @@ fun AgentDemoLoadedScreen( item { StatusIndicator( status = uiState.status, - packageManager = packageManager + packageManager = packageManager, ) } } items( items = uiState.messages.reversed(), - key = { message -> message.messageId } + key = { message -> message.messageId }, ) { message -> MessageBubble( message = message, isValidAction = - message.pendingIntentId in uiState.activePendingActionIds, + message.pendingIntentId in uiState.activePendingActionIds, installedApps = uiState.installedApps, - onConfirmAction = { onEvent(AgentUiEvent.OnConfirmAction(it)) } + onConfirmAction = { onEvent(AgentUiEvent.OnConfirmAction(it)) }, ) } } @@ -378,12 +378,12 @@ fun AgentDemoLoadedScreen( anchorBounds: IntRect, windowSize: IntSize, layoutDirection: LayoutDirection, - popupContentSize: IntSize + popupContentSize: IntSize, ): IntOffset { val gap = with(density) { 2.dp.roundToPx() } return IntOffset( x = anchorBounds.left, - y = anchorBounds.top - popupContentSize.height - gap + y = anchorBounds.top - popupContentSize.height - gap, ) } } @@ -414,61 +414,61 @@ fun AgentDemoLoadedScreen( } }, modifier = - Modifier - .fillMaxWidth() - .padding(vertical = 16.dp) - .onPreviewKeyEvent { keyEvent -> - if ( - (keyEvent.key == Key.Enter || keyEvent.key == Key.NumPadEnter) && - keyEvent.type == KeyEventType.KeyDown - ) { - sendMessage() - true - } else { - false - } - }, + Modifier + .fillMaxWidth() + .padding(vertical = 16.dp) + .onPreviewKeyEvent { keyEvent -> + if ( + (keyEvent.key == Key.Enter || keyEvent.key == Key.NumPadEnter) && + keyEvent.type == KeyEventType.KeyDown + ) { + sendMessage() + true + } else { + false + } + }, enabled = uiState.status == AgentStatus.Idle, shape = CircleShape, placeholder = { Text(stringResource(R.string.agent_demo_ask_agent)) }, visualTransformation = visualTransformation, colors = - OutlinedTextFieldDefaults.colors( - unfocusedContainerColor = MaterialTheme.colorScheme.surfaceBright, - focusedContainerColor = MaterialTheme.colorScheme.surfaceBright, - unfocusedBorderColor = Color.Transparent, - focusedBorderColor = Color.Transparent - ), + OutlinedTextFieldDefaults.colors( + unfocusedContainerColor = MaterialTheme.colorScheme.surfaceBright, + focusedContainerColor = MaterialTheme.colorScheme.surfaceBright, + unfocusedBorderColor = Color.Transparent, + focusedBorderColor = Color.Transparent, + ), trailingIcon = { IconButton( onClick = sendMessage, enabled = - messageText.text.isNotBlank() && - uiState.status == AgentStatus.Idle + messageText.text.isNotBlank() && + uiState.status == AgentStatus.Idle, ) { Icon( imageVector = Icons.AutoMirrored.Filled.Send, contentDescription = - stringResource(R.string.agent_demo_send) + stringResource(R.string.agent_demo_send), ) } - } + }, ) if (showAutocomplete && filteredApps.isNotEmpty()) { Popup( popupPositionProvider = popupPositionProvider, onDismissRequest = {}, - properties = PopupProperties(focusable = false) + properties = PopupProperties(focusable = false), ) { Card( modifier = Modifier.fillMaxWidth(0.9f), elevation = CardDefaults.cardElevation(defaultElevation = 8.dp), colors = - CardDefaults.cardColors( - containerColor = MaterialTheme.colorScheme.surfaceBright - ), - shape = MaterialTheme.shapes.medium + CardDefaults.cardColors( + containerColor = MaterialTheme.colorScheme.surfaceBright, + ), + shape = MaterialTheme.shapes.medium, ) { Column(modifier = Modifier.padding(vertical = 4.dp)) { filteredApps.take(5).forEach { app -> @@ -479,18 +479,18 @@ fun AgentDemoLoadedScreen( val selectionStart = messageText.selection.start val textBeforeCursor = currentText.take( - selectionStart + selectionStart, ) val textAfterCursor = currentText.drop( - selectionStart + selectionStart, ) val mentionIndex = textBeforeCursor.lastIndexOf('@') if (mentionIndex >= 0) { val textBeforeMention = textBeforeCursor.substring( 0, - mentionIndex + mentionIndex, ) val newText = "$textBeforeMention@${app.label} $textAfterCursor" @@ -500,13 +500,13 @@ fun AgentDemoLoadedScreen( TextFieldValue( text = newText, selection = - TextRange( - newCursorPosition - ) + TextRange( + newCursorPosition, + ), ) selectedAppPackageName = app.packageName } - } + }, ) } } @@ -525,19 +525,19 @@ fun ModelDropdown( modifier: Modifier = Modifier, currentThread: ThreadEntity?, onModelSelected: (LlmModel) -> Unit, - onMenuClick: () -> Unit + onMenuClick: () -> Unit, ) { var expanded by remember { mutableStateOf(false) } ExposedDropdownMenuBox( modifier = modifier, expanded = expanded, - onExpandedChange = { expanded = !expanded } + onExpandedChange = { expanded = !expanded }, ) { Surface( modifier = Modifier.padding(bottom = 8.dp), shadowElevation = 2.dp, shape = CircleShape, - color = MaterialTheme.colorScheme.surfaceBright + color = MaterialTheme.colorScheme.surfaceBright, ) { val text = currentThread?.llmModel?.modelName @@ -551,38 +551,38 @@ fun ModelDropdown( Row( modifier = - Modifier - .fillMaxWidth() - .height(56.dp) - .padding(start = 4.dp, end = 16.dp), - verticalAlignment = Alignment.CenterVertically + Modifier + .fillMaxWidth() + .height(56.dp) + .padding(start = 4.dp, end = 16.dp), + verticalAlignment = Alignment.CenterVertically, ) { IconButton(onClick = onMenuClick) { Icon(imageVector = Icons.Default.Menu, contentDescription = "Menu") } Row( modifier = - Modifier - .weight(1f) - .fillMaxHeight() - .menuAnchor( - ExposedDropdownMenuAnchorType.PrimaryEditable, - enabled = true - ), - verticalAlignment = Alignment.CenterVertically + Modifier + .weight(1f) + .fillMaxHeight() + .menuAnchor( + ExposedDropdownMenuAnchorType.PrimaryEditable, + enabled = true, + ), + verticalAlignment = Alignment.CenterVertically, ) { Column(modifier = Modifier.weight(1f)) { Text( text = stringResource(R.string.agent_demo_title), style = MaterialTheme.typography.labelMedium, - color = MaterialTheme.colorScheme.onSurfaceVariant + color = MaterialTheme.colorScheme.onSurfaceVariant, ) Text( text = text, style = MaterialTheme.typography.bodyMedium, color = textColor, maxLines = 1, - overflow = TextOverflow.Ellipsis + overflow = TextOverflow.Ellipsis, ) } Icon(imageVector = Icons.Default.ArrowDropDown, contentDescription = null) @@ -595,21 +595,21 @@ fun ModelDropdown( onDismissRequest = { expanded = false }, modifier = Modifier.exposedDropdownSize(), containerColor = MaterialTheme.colorScheme.surfaceBright, - shape = RoundedCornerShape(28.dp) + shape = RoundedCornerShape(28.dp), ) { item { Text( "--- Gemini ---", color = MaterialTheme.colorScheme.secondary, modifier = Modifier.padding(horizontal = 24.dp, vertical = 8.dp), - style = MaterialTheme.typography.labelLarge + style = MaterialTheme.typography.labelLarge, ) } val models = listOf( LlmModel.GEMINI_3_1_PRO_PREVIEW, LlmModel.GEMINI_3_FLASH_PREVIEW, - LlmModel.GEMINI_3_1_FLASH_LITE_PREVIEW + LlmModel.GEMINI_3_1_FLASH_LITE_PREVIEW, ) items(models) { model -> DropdownMenuItem( @@ -618,7 +618,7 @@ fun ModelDropdown( onModelSelected(model) expanded = false }, - contentPadding = PaddingValues(horizontal = 24.dp, vertical = 8.dp) + contentPadding = PaddingValues(horizontal = 24.dp, vertical = 8.dp), ) } } @@ -630,7 +630,7 @@ fun MessageBubble( message: MessageEntity, isValidAction: Boolean, installedApps: List, - onConfirmAction: (String) -> Unit + onConfirmAction: (String) -> Unit, ) { val alignment = if (message.role == MessageRole.USER) Alignment.End else Alignment.Start val isError = message.processingStatus == MessageProcessingStatus.FAILED @@ -649,15 +649,15 @@ fun MessageBubble( Column( modifier = - Modifier - .fillMaxWidth() - .padding(vertical = 4.dp, horizontal = 2.dp), - horizontalAlignment = alignment + Modifier + .fillMaxWidth() + .padding(vertical = 4.dp, horizontal = 2.dp), + horizontalAlignment = alignment, ) { Surface( shape = MaterialTheme.shapes.large, color = backgroundColor, - shadowElevation = if (message.role == MessageRole.ASSISTANT) 1.dp else 0.dp + shadowElevation = if (message.role == MessageRole.ASSISTANT) 1.dp else 0.dp, ) { Column(modifier = Modifier.padding(12.dp)) { SelectionContainer { @@ -666,7 +666,7 @@ fun MessageBubble( Icon( imageVector = Icons.Filled.Warning, contentDescription = stringResource(R.string.debugging_error), - tint = textColor + tint = textColor, ) Spacer(modifier = Modifier.width(8.dp)) } @@ -701,18 +701,18 @@ fun MessageBubble( installedApps, chipBgColor, chipTextColor, - density + density, ) { val map = mutableMapOf() if (installedApps.isNotEmpty() && contentText.contains("@")) { val appLabelsPattern = installedApps.joinToString( - "|" + "|", ) { Regex.escape(it.label) } val regex = Regex( "@($appLabelsPattern)\\b", - RegexOption.IGNORE_CASE + RegexOption.IGNORE_CASE, ) regex.findAll(contentText).forEachIndexed { index, match -> val id = "chip_$index" @@ -721,17 +721,17 @@ fun MessageBubble( textMeasurer.measure( text = appName, style = - typographyStyle.copy( - fontWeight = FontWeight.Bold - ) + typographyStyle.copy( + fontWeight = FontWeight.Bold, + ), ) val widthSp = with( - density + density, ) { (measured.size.width + 8.dp.roundToPx()).toSp() } val heightSp = with( - density + density, ) { (measured.size.height + 2.dp.roundToPx()).toSp() } map[id] = @@ -739,29 +739,29 @@ fun MessageBubble( Placeholder( width = widthSp, height = heightSp, - placeholderVerticalAlign = PlaceholderVerticalAlign.TextCenter - ) + placeholderVerticalAlign = PlaceholderVerticalAlign.TextCenter, + ), ) { Surface( shape = - androidx.compose.foundation.shape.RoundedCornerShape( - 6.dp - ), - color = chipBgColor + androidx.compose.foundation.shape.RoundedCornerShape( + 6.dp, + ), + color = chipBgColor, ) { Box(contentAlignment = Alignment.Center) { Text( text = appName, color = chipTextColor, style = - typographyStyle.copy( - fontWeight = FontWeight.Bold - ), + typographyStyle.copy( + fontWeight = FontWeight.Bold, + ), modifier = - Modifier.padding( - horizontal = 4.dp, - vertical = 1.dp - ) + Modifier.padding( + horizontal = 4.dp, + vertical = 1.dp, + ), ) } } @@ -775,7 +775,7 @@ fun MessageBubble( text = formattedText, inlineContent = inlineContentMap, color = textColor, - style = typographyStyle + style = typographyStyle, ) } } @@ -788,17 +788,17 @@ fun MessageBubble( enabled = isValidAction, shape = CircleShape, colors = - ButtonDefaults.buttonColors( - containerColor = MaterialTheme.colorScheme.primary, - contentColor = MaterialTheme.colorScheme.onPrimary - ) + ButtonDefaults.buttonColors( + containerColor = MaterialTheme.colorScheme.primary, + contentColor = MaterialTheme.colorScheme.onPrimary, + ), ) { Text( if (isValidAction) { stringResource(R.string.agent_demo_confirm_action) } else { stringResource(R.string.agent_demo_action_expired) - } + }, ) } } @@ -813,11 +813,11 @@ fun MessageBubble( model = attachment.uri, contentDescription = "Generated Image", modifier = - Modifier - .fillMaxWidth(0.85f) - .height(240.dp) - .clip(RoundedCornerShape(16.dp)), - contentScale = ContentScale.Crop + Modifier + .fillMaxWidth(0.85f) + .height(240.dp) + .clip(RoundedCornerShape(16.dp)), + contentScale = ContentScale.Crop, ) } } @@ -826,21 +826,24 @@ fun MessageBubble( } @Composable -fun StatusIndicator(status: AgentStatus, packageManager: PackageManager) { +fun StatusIndicator( + status: AgentStatus, + packageManager: PackageManager, +) { when (status) { AgentStatus.Thinking -> { Row( modifier = - Modifier - .fillMaxWidth() - .padding(vertical = 8.dp), - verticalAlignment = Alignment.CenterVertically + Modifier + .fillMaxWidth() + .padding(vertical = 8.dp), + verticalAlignment = Alignment.CenterVertically, ) { CircularProgressIndicator(modifier = Modifier.size(24.dp)) Spacer(modifier = Modifier.width(8.dp)) Text( stringResource(R.string.agent_demo_thinking), - style = MaterialTheme.typography.bodyMedium + style = MaterialTheme.typography.bodyMedium, ) } } @@ -862,22 +865,22 @@ fun StatusIndicator(status: AgentStatus, packageManager: PackageManager) { Surface( modifier = - Modifier - .fillMaxWidth() - .padding(vertical = 8.dp), + Modifier + .fillMaxWidth() + .padding(vertical = 8.dp), shape = MaterialTheme.shapes.large, color = MaterialTheme.colorScheme.surfaceBright, - shadowElevation = 2.dp + shadowElevation = 2.dp, ) { Row( modifier = Modifier.padding(12.dp), - verticalAlignment = Alignment.CenterVertically + verticalAlignment = Alignment.CenterVertically, ) { appIcon?.let { Image( bitmap = it.toBitmap().asImageBitmap(), contentDescription = null, - modifier = Modifier.size(40.dp) + modifier = Modifier.size(40.dp), ) Spacer(modifier = Modifier.width(12.dp)) } @@ -888,7 +891,7 @@ fun StatusIndicator(status: AgentStatus, packageManager: PackageManager) { Spacer(modifier = Modifier.width(8.dp)) Text( stringResource(R.string.agent_demo_connecting), - style = MaterialTheme.typography.bodyMedium + style = MaterialTheme.typography.bodyMedium, ) } } @@ -907,26 +910,26 @@ fun ChatHistorySidePanel( threads: List, currentThread: ThreadEntity?, onEvent: (AgentUiEvent) -> Unit, - modifier: Modifier = Modifier + modifier: Modifier = Modifier, ) { Column( modifier = - modifier - .width(280.dp) - .fillMaxHeight() - .padding(16.dp) + modifier + .width(280.dp) + .fillMaxHeight() + .padding(16.dp), ) { Text( text = stringResource(R.string.agent_demo_chat_history), style = MaterialTheme.typography.titleLarge, color = MaterialTheme.colorScheme.onSurface, - modifier = Modifier.padding(bottom = 16.dp) + modifier = Modifier.padding(bottom = 16.dp), ) LazyColumn(modifier = Modifier.fillMaxSize()) { items( items = threads, - key = { thread -> thread.threadId } + key = { thread -> thread.threadId }, ) { thread -> val isSelected = thread.threadId == currentThread?.threadId val backgroundColor = @@ -944,26 +947,26 @@ fun ChatHistorySidePanel( Surface( modifier = - Modifier - .fillMaxWidth() - .padding(vertical = 4.dp) - .clickable { - onEvent(AgentUiEvent.OnThreadSelected(thread.threadId)) - }, + Modifier + .fillMaxWidth() + .padding(vertical = 4.dp) + .clickable { + onEvent(AgentUiEvent.OnThreadSelected(thread.threadId)) + }, shape = MaterialTheme.shapes.medium, color = backgroundColor, - contentColor = textColor + contentColor = textColor, ) { Column(modifier = Modifier.padding(12.dp)) { Text( text = thread.llmModel.modelName, style = MaterialTheme.typography.bodyMedium, - color = textColor + color = textColor, ) Text( text = "ID: ${thread.threadId.take(8)}", style = MaterialTheme.typography.bodySmall, - color = textColor.copy(alpha = 0.7f) + color = textColor.copy(alpha = 0.7f), ) } } @@ -974,7 +977,7 @@ fun ChatHistorySidePanel( class InlineAppScopingVisualTransformation( private val installedApps: List, - private val chipTextColor: Color + private val chipTextColor: Color, ) : VisualTransformation { private val regex: Regex? = if (installedApps.isNotEmpty()) { @@ -1002,8 +1005,8 @@ class InlineAppScopingVisualTransformation( withStyle( SpanStyle( color = chipTextColor, - fontWeight = FontWeight.Bold - ) + fontWeight = FontWeight.Bold, + ), ) { append(match.value) } @@ -1018,7 +1021,10 @@ class InlineAppScopingVisualTransformation( } } -fun formatMessageText(text: String, installedApps: List): AnnotatedString { +fun formatMessageText( + text: String, + installedApps: List, +): AnnotatedString { if (installedApps.isEmpty() || !text.contains("@")) { return AnnotatedString(text) } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/debugging/FunctionsFoundContent.kt b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/debugging/FunctionsFoundContent.kt index 146d1f6..ca95ccb 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/debugging/FunctionsFoundContent.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/ui/screens/debugging/FunctionsFoundContent.kt @@ -65,7 +65,6 @@ import androidx.compose.ui.text.font.FontWeight import androidx.compose.ui.text.withStyle import androidx.compose.ui.tooling.preview.Preview import androidx.compose.ui.unit.dp -import androidx.appfunctions.AppFunctionException import com.example.appfunctions.agent.R import com.example.appfunctions.agent.domain.appfunction.AppFunctionExceptionFormatter import com.example.appfunctions.agent.domain.appfunction.ExecuteAppFunctionResult diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/data/db/entities/MessageAttachmentConverterTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/data/db/entities/MessageAttachmentConverterTest.kt index 5c665cb..7a7994e 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/data/db/entities/MessageAttachmentConverterTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/data/db/entities/MessageAttachmentConverterTest.kt @@ -20,7 +20,6 @@ import org.junit.Assert.assertTrue import org.junit.Test class MessageAttachmentConverterTest { - private val converter = MessageAttachmentConverter() @Test @@ -29,12 +28,12 @@ class MessageAttachmentConverterTest { listOf( MessageAttachment( uri = "content://com.example.appfunctions.agent.fileprovider/cache/test.jpg", - mimeType = "image/jpeg" + mimeType = "image/jpeg", ), MessageAttachment( uri = "content://com.example.appfunctions.agent.fileprovider/cache/test.png", - mimeType = "image/png" - ) + mimeType = "image/png", + ), ) val json = converter.fromAttachments(attachments) diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt index b6f3688..b8a288f 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt @@ -22,6 +22,7 @@ import androidx.appfunctions.metadata.AppFunctionPackageMetadata import com.example.appfunctions.agent.data.LlmModel import com.example.appfunctions.agent.data.LlmProviderName import com.example.appfunctions.agent.data.SettingsRepository +import com.example.appfunctions.agent.data.db.entities.MessageAttachment import com.example.appfunctions.agent.data.db.entities.MessageEntity import com.example.appfunctions.agent.data.db.entities.MessageProcessingStatus import com.example.appfunctions.agent.data.db.entities.MessageRole @@ -86,145 +87,149 @@ class AgentOrchestratorTest { getAppFunctionsUseCase = getAppFunctionsUseCase, convertInputToAppFunctionDataUseCase = convertInputToAppFunctionDataUseCase, executeAppFunctionUseCase = executeAppFunctionUseCase, - savePendingIntentUseCase = savePendingIntentUseCase + savePendingIntentUseCase = savePendingIntentUseCase, ) } @Test - fun `observeAndProcessMessages fails when API key is missing`() = runTest { - val threadId = "thread_1" - val message = createUserMessage(threadId, "Hello", messageId = "msg_1") - val thread = createThread(threadId) + fun `observeAndProcessMessages fails when API key is missing`() = + runTest { + val threadId = "thread_1" + val message = createUserMessage(threadId, "Hello", messageId = "msg_1") + val thread = createThread(threadId) - setupDefaultMocks(threadId, message, thread, apiKey = null) + setupDefaultMocks(threadId, message, thread, apiKey = null) - agentOrchestrator.observeAndProcessMessages(threadId) + agentOrchestrator.observeAndProcessMessages(threadId) - // Verify behavior (state) - assertEquals(AgentStatus.Idle, agentOrchestrator.status.value) + // Verify behavior (state) + assertEquals(AgentStatus.Idle, agentOrchestrator.status.value) - // Verify interactions - coVerify { - updateMessageUseCase( - message.messageId, - UpdateMessageParams(status = MessageProcessingStatus.PROCESSED) - ) - sendMessageUseCase( - threadId = threadId, - role = MessageRole.ASSISTANT, - textContent = "API key is missing for GEMINI", - processingStatus = MessageProcessingStatus.FAILED - ) + // Verify interactions + coVerify { + updateMessageUseCase( + message.messageId, + UpdateMessageParams(status = MessageProcessingStatus.PROCESSED), + ) + sendMessageUseCase( + threadId = threadId, + role = MessageRole.ASSISTANT, + textContent = "API key is missing for GEMINI", + processingStatus = MessageProcessingStatus.FAILED, + ) + } } - } @Test - fun `observeAndProcessMessages fails when LLM returns error`() = runTest { - val threadId = "thread_1" - val message = createUserMessage(threadId, "Hello", messageId = "msg_1") - val thread = createThread(threadId) - val llmProvider = mockk() + fun `observeAndProcessMessages fails when LLM returns error`() = + runTest { + val threadId = "thread_1" + val message = createUserMessage(threadId, "Hello", messageId = "msg_1") + val thread = createThread(threadId) + val llmProvider = mockk() - setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) - coEvery { getAppFunctionsUseCase() } returns flowOf(emptyMap()) + setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) + coEvery { getAppFunctionsUseCase() } returns flowOf(emptyMap()) - val errorMsg = "LLM failed" - coEvery { llmProvider.generateResponse(any(), any(), any(), any(), any()) } returns - LlmResponse.Error(errorMsg) + val errorMsg = "LLM failed" + coEvery { llmProvider.generateResponse(any(), any(), any(), any(), any()) } returns + LlmResponse.Error(errorMsg) - agentOrchestrator.observeAndProcessMessages(threadId) + agentOrchestrator.observeAndProcessMessages(threadId) - // Verify behavior (state) - assertEquals(AgentStatus.Idle, agentOrchestrator.status.value) + // Verify behavior (state) + assertEquals(AgentStatus.Idle, agentOrchestrator.status.value) - // Verify interactions - coVerify { - updateMessageUseCase( - message.messageId, - UpdateMessageParams(status = MessageProcessingStatus.PROCESSED) - ) - sendMessageUseCase( - threadId = threadId, - role = MessageRole.ASSISTANT, - textContent = errorMsg, - processingStatus = MessageProcessingStatus.FAILED - ) + // Verify interactions + coVerify { + updateMessageUseCase( + message.messageId, + UpdateMessageParams(status = MessageProcessingStatus.PROCESSED), + ) + sendMessageUseCase( + threadId = threadId, + role = MessageRole.ASSISTANT, + textContent = errorMsg, + processingStatus = MessageProcessingStatus.FAILED, + ) + } } - } @Test - fun `observeAndProcessMessages succeeds when LLM returns text`() = runTest { - val threadId = "thread_1" - val message = createUserMessage(threadId, "Hello", messageId = "msg_1") - val thread = createThread(threadId) - val llmProvider = mockk() - - setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) - coEvery { getAppFunctionsUseCase() } returns flowOf(emptyMap()) - - val responseText = "Hi there" - coEvery { llmProvider.generateResponse(any(), any(), any(), any(), any()) } returns - LlmResponse.Success( - "interaction_123", - listOf(LlmResponsePart.Text(responseText)) - ) + fun `observeAndProcessMessages succeeds when LLM returns text`() = + runTest { + val threadId = "thread_1" + val message = createUserMessage(threadId, "Hello", messageId = "msg_1") + val thread = createThread(threadId) + val llmProvider = mockk() - agentOrchestrator.observeAndProcessMessages(threadId) + setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) + coEvery { getAppFunctionsUseCase() } returns flowOf(emptyMap()) - // Verify behavior (state) - assertEquals(AgentStatus.Idle, agentOrchestrator.status.value) + val responseText = "Hi there" + coEvery { llmProvider.generateResponse(any(), any(), any(), any(), any()) } returns + LlmResponse.Success( + "interaction_123", + listOf(LlmResponsePart.Text(responseText)), + ) - // Verify interactions - coVerify { - updateThreadUseCase(threadId, UpdateThreadParams(interactionId = "interaction_123")) - sendMessageUseCase( - threadId = threadId, - role = MessageRole.ASSISTANT, - textContent = responseText, - processingStatus = MessageProcessingStatus.PROCESSED - ) - updateMessageUseCase( - message.messageId, - UpdateMessageParams(status = MessageProcessingStatus.PROCESSED) - ) + agentOrchestrator.observeAndProcessMessages(threadId) + + // Verify behavior (state) + assertEquals(AgentStatus.Idle, agentOrchestrator.status.value) + + // Verify interactions + coVerify { + updateThreadUseCase(threadId, UpdateThreadParams(interactionId = "interaction_123")) + sendMessageUseCase( + threadId = threadId, + role = MessageRole.ASSISTANT, + textContent = responseText, + processingStatus = MessageProcessingStatus.PROCESSED, + ) + updateMessageUseCase( + message.messageId, + UpdateMessageParams(status = MessageProcessingStatus.PROCESSED), + ) + } } - } @Test - fun `observeAndProcessMessages scopes tools when targetPackageName is set`() = runTest { - val threadId = "thread_1" - val message = - createUserMessage( - threadId = threadId, - textContent = "run geo code address for n1c4ag", - targetPackageName = "com.google.android.appfunctiontestingagent" - ) - val thread = createThread(threadId) - val llmProvider = mockk() + fun `observeAndProcessMessages scopes tools when targetPackageName is set`() = + runTest { + val threadId = "thread_1" + val message = + createUserMessage( + threadId = threadId, + textContent = "run geo code address for n1c4ag", + targetPackageName = "com.google.android.appfunctiontestingagent", + ) + val thread = createThread(threadId) + val llmProvider = mockk() - val tool1 = createMockTool("com.google.android.appfunctiontestingagent", "run_geo_code") - val tool2 = - createMockTool("com.google.android.digitalwellbeing", "digital_well_being_tool") - mockAppFunctions(listOf(tool1, tool2)) + val tool1 = createMockTool("com.google.android.appfunctiontestingagent", "run_geo_code") + val tool2 = + createMockTool("com.google.android.digitalwellbeing", "digital_well_being_tool") + mockAppFunctions(listOf(tool1, tool2)) - setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) + setupDefaultMocks(threadId, message, thread, llmProvider = llmProvider) - coEvery { - llmProvider.generateResponse(any(), any(), any(), any(), any()) - } returns LlmResponse.Success("interaction_id", listOf(LlmResponsePart.Text("Success"))) + coEvery { + llmProvider.generateResponse(any(), any(), any(), any(), any()) + } returns LlmResponse.Success("interaction_id", listOf(LlmResponsePart.Text("Success"))) - agentOrchestrator.observeAndProcessMessages(threadId) + agentOrchestrator.observeAndProcessMessages(threadId) - coVerify { - llmProvider.generateResponse( - previousInteractionId = null, - input = eq(LlmInput.UserMessage("run geo code address for n1c4ag")), - tools = listOf(tool1), - apiKey = "dummy_key", - modelName = any() - ) + coVerify { + llmProvider.generateResponse( + previousInteractionId = null, + input = eq(LlmInput.UserMessage("run geo code address for n1c4ag")), + tools = listOf(tool1), + apiKey = "dummy_key", + modelName = any(), + ) + } } - } @Test fun `observeAndProcessMessages does not scope tools when targetPackageName is null`() = @@ -253,7 +258,7 @@ class AgentOrchestratorTest { input = eq(LlmInput.UserMessage("run geo code address for n1c4ag")), tools = listOf(tool1, tool2), apiKey = "dummy_key", - modelName = any() + modelName = any(), ) } } @@ -284,17 +289,18 @@ class AgentOrchestratorTest { coEvery { llmProvider.generateResponse(null, any(), any(), any(), any()) - } returns LlmResponse.Success( - "interaction_1", - listOf( - LlmResponsePart.ToolCall( - packageName = "com.example.calendar", - functionId = "create_event", - arguments = emptyMap(), - callId = "call_1", - ) + } returns + LlmResponse.Success( + "interaction_1", + listOf( + LlmResponsePart.ToolCall( + packageName = "com.example.calendar", + functionId = "create_event", + arguments = emptyMap(), + callId = "call_1", + ), + ), ) - ) val expectedErrorOutput = "Tool execution failed for create_event: Error: AppFunctionPermissionRequiredException - Calendar permission required" @@ -307,17 +313,18 @@ class AgentOrchestratorTest { coVerify { llmProvider.generateResponse( previousInteractionId = eq("interaction_1"), - input = eq( - LlmInput.ToolResponse( - listOf( - ToolOutput( - functionId = "create_event", - callId = "call_1", - result = expectedErrorOutput, - ) - ) - ) - ), + input = + eq( + LlmInput.ToolResponse( + listOf( + ToolOutput( + functionId = "create_event", + callId = "call_1", + result = expectedErrorOutput, + ), + ), + ), + ), tools = listOf(tool1), apiKey = "dummy_key", modelName = any(), @@ -329,7 +336,7 @@ class AgentOrchestratorTest { threadId: String, textContent: String, messageId: String = "message_1", - targetPackageName: String? = null + targetPackageName: String? = null, ) = MessageEntity( messageId = messageId, threadId = threadId, @@ -337,24 +344,24 @@ class AgentOrchestratorTest { textContent = textContent, timestamp = System.currentTimeMillis(), processingStatus = MessageProcessingStatus.PENDING_AGENT_RESPONSE, - targetPackageName = targetPackageName + targetPackageName = targetPackageName, ) private fun createThread( threadId: String, llmModel: LlmModel = LlmModel.GEMINI_3_FLASH_PREVIEW, - latestInteractionId: String? = null + latestInteractionId: String? = null, ) = ThreadEntity( threadId = threadId, createdAt = System.currentTimeMillis(), llmModel = llmModel, - latestInteractionId = latestInteractionId + latestInteractionId = latestInteractionId, ) private fun createMockTool( packageName: String, id: String, - isEnabled: Boolean = true + isEnabled: Boolean = true, ): AppFunctionMetadata { val tool = mockk(relaxed = true) every { tool.packageName } returns packageName @@ -374,7 +381,7 @@ class AgentOrchestratorTest { thread: ThreadEntity, apiKey: String? = "dummy_key", disconnectedApps: Set = emptySet(), - llmProvider: LlmProvider = mockk() + llmProvider: LlmProvider = mockk(), ) { coEvery { observePendingMessagesUseCase(threadId) } returns flow { @@ -404,18 +411,18 @@ class AgentOrchestratorTest { packageName = "com.example.appfunctions.agent", functionId = "generateImage", arguments = mapOf("prompt" to "dog"), - callId = "call_1" + callId = "call_1", ) val firstResponse = LlmResponse.Success( interactionId = "interaction_1", - parts = listOf(toolCall) + parts = listOf(toolCall), ) val secondResponse = LlmResponse.Success( interactionId = "interaction_2", - parts = listOf(LlmResponsePart.Text("Here is your image!")) + parts = listOf(LlmResponsePart.Text("Here is your image!")), ) coEvery { @@ -424,7 +431,7 @@ class AgentOrchestratorTest { input = any(), tools = any(), apiKey = any(), - modelName = any() + modelName = any(), ) } returns firstResponse @@ -434,7 +441,7 @@ class AgentOrchestratorTest { input = any(), tools = any(), apiKey = any(), - modelName = any() + modelName = any(), ) } returns secondResponse @@ -447,11 +454,15 @@ class AgentOrchestratorTest { } returns ExecuteAppFunctionResult.Data( data = AppFunctionData.EMPTY, - formattedJson = """{"imageUri":"content://com.example.appfunctions.agent.fileprovider/cache/test.jpg","mimeType":"image/jpeg","prompt":"dog"}""" + formattedJson = + """{"imageUri":"content://com.example.appfunctions.agent.fileprovider/cache/test.jpg",""" + + """"mimeType":"image/jpeg","prompt":"dog"}""", ) agentOrchestrator.observeAndProcessMessages(threadId) + val testUri = + "content://com.example.appfunctions.agent.fileprovider/cache/test.jpg" coVerify { sendMessageUseCase( threadId = threadId, @@ -460,13 +471,7 @@ class AgentOrchestratorTest { processingStatus = MessageProcessingStatus.PROCESSED, pendingIntentId = null, targetPackageName = null, - attachments = - listOf( - com.example.appfunctions.agent.data.db.entities.MessageAttachment( - uri = "content://com.example.appfunctions.agent.fileprovider/cache/test.jpg", - mimeType = "image/jpeg" - ) - ) + attachments = listOf(MessageAttachment(uri = testUri, mimeType = "image/jpeg")), ) } } @@ -489,7 +494,7 @@ class AgentOrchestratorTest { packageName = "com.example.targetapp", functionId = "setWallpaper", arguments = mapOf("uri" to contentUri), - callId = "call_2" + callId = "call_2", ) coEvery { @@ -498,12 +503,12 @@ class AgentOrchestratorTest { input = any(), tools = any(), apiKey = any(), - modelName = any() + modelName = any(), ) } returns LlmResponse.Success( interactionId = "interaction_1", - parts = listOf(toolCall) + parts = listOf(toolCall), ) coEvery { @@ -512,12 +517,12 @@ class AgentOrchestratorTest { input = any(), tools = any(), apiKey = any(), - modelName = any() + modelName = any(), ) } returns LlmResponse.Success( interactionId = "interaction_2", - parts = listOf(LlmResponsePart.Text("Wallpaper set")) + parts = listOf(LlmResponsePart.Text("Wallpaper set")), ) coEvery { @@ -529,7 +534,7 @@ class AgentOrchestratorTest { } returns ExecuteAppFunctionResult.Data( data = AppFunctionData.EMPTY, - formattedJson = """{"success":true}""" + formattedJson = """{"success":true}""", ) agentOrchestrator.observeAndProcessMessages(threadId) @@ -538,7 +543,7 @@ class AgentOrchestratorTest { context.grantUriPermission( eq("com.example.targetapp"), any(), - eq(Intent.FLAG_GRANT_READ_URI_PERMISSION) + eq(Intent.FLAG_GRANT_READ_URI_PERMISSION), ) } } diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/AppFunctionExceptionFormatterTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/AppFunctionExceptionFormatterTest.kt index b60f919..906a3dd 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/AppFunctionExceptionFormatterTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/AppFunctionExceptionFormatterTest.kt @@ -86,7 +86,8 @@ class AppFunctionExceptionFormatterTest { "com.example.chatapp.appfunctions.BaseChatAppFunctionService#send", ) assertEquals( - "Tool execution failed for com.example.chatapp.appfunctions.BaseChatAppFunctionService#send: Error: AppFunctionInvalidArgumentException - Message body cannot be empty", + "Tool execution failed for com.example.chatapp.appfunctions.BaseChatAppFunctionService#send: " + + "Error: AppFunctionInvalidArgumentException - Message body cannot be empty", formatted, ) } diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ExecuteAppFunctionUseCaseTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ExecuteAppFunctionUseCaseTest.kt index 7d4db46..9c660ff 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ExecuteAppFunctionUseCaseTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ExecuteAppFunctionUseCaseTest.kt @@ -38,38 +38,39 @@ class ExecuteAppFunctionUseCaseTest { private val convertAppFunctionDataToJsonUseCase = mockk() private val useCase = ExecuteAppFunctionUseCase(appFunctionManager, convertAppFunctionDataToJsonUseCase) - @Test - fun `invoke returns Error with original AppFunctionException when response is Error`() = runTest { - val function = mockk(relaxed = true) - every { function.packageName } returns "com.example.app" - every { function.id } returns "test_function" - val parameters = mockk() + fun `invoke returns Error with original AppFunctionException when response is Error`() = + runTest { + val function = mockk(relaxed = true) + every { function.packageName } returns "com.example.app" + every { function.id } returns "test_function" + val parameters = mockk() - val appException = AppFunctionPermissionRequiredException("Permission needed") - coEvery { appFunctionManager.executeAppFunction(any()) } returns ExecuteAppFunctionResponse.Error(appException) + val appException = AppFunctionPermissionRequiredException("Permission needed") + coEvery { appFunctionManager.executeAppFunction(any()) } returns ExecuteAppFunctionResponse.Error(appException) - val result = useCase(function, parameters) + val result = useCase(function, parameters) - assertTrue(result is ExecuteAppFunctionResult.Error) - val errorResult = result as ExecuteAppFunctionResult.Error - assertEquals(appException, errorResult.exception) - } + assertTrue(result is ExecuteAppFunctionResult.Error) + val errorResult = result as ExecuteAppFunctionResult.Error + assertEquals(appException, errorResult.exception) + } @Test - fun `invoke returns Error when executeAppFunction throws exception`() = runTest { - val function = mockk(relaxed = true) - every { function.packageName } returns "com.example.app" - every { function.id } returns "test_function" - val parameters = mockk() + fun `invoke returns Error when executeAppFunction throws exception`() = + runTest { + val function = mockk(relaxed = true) + every { function.packageName } returns "com.example.app" + every { function.id } returns "test_function" + val parameters = mockk() - val runtimeException = RuntimeException("Unexpected crash") - coEvery { appFunctionManager.executeAppFunction(any()) } throws runtimeException + val runtimeException = RuntimeException("Unexpected crash") + coEvery { appFunctionManager.executeAppFunction(any()) } throws runtimeException - val result = useCase(function, parameters) + val result = useCase(function, parameters) - assertTrue(result is ExecuteAppFunctionResult.Error) - val errorResult = result as ExecuteAppFunctionResult.Error - assertEquals(runtimeException, errorResult.exception) - } + assertTrue(result is ExecuteAppFunctionResult.Error) + val errorResult = result as ExecuteAppFunctionResult.Error + assertEquals(runtimeException, errorResult.exception) + } } From d678128f6b61818ed8c2d7c8a0659df8712cb301 Mon Sep 17 00:00:00 2001 From: Caleb Areeveso Date: Wed, 15 Jul 2026 09:34:20 +0000 Subject: [PATCH 09/16] refactor: Improve kotlin-readbility and break down generateImage into private helpers Change-Id: I484c7cb582b8cd436fee8793119b434aa3d84784 --- .../data/BaseBuiltInAppFunctionService.kt | 252 ++++++++++-------- 1 file changed, 140 insertions(+), 112 deletions(-) diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/BaseBuiltInAppFunctionService.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/BaseBuiltInAppFunctionService.kt index 912426e..2e1eb54 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/BaseBuiltInAppFunctionService.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/BaseBuiltInAppFunctionService.kt @@ -178,146 +178,174 @@ abstract class BaseBuiltInAppFunctionService : AppFunctionService() { aspectRatio: String? = null, ): GeneratedImageResult = withContext(Dispatchers.IO) { - val context = this@BaseBuiltInAppFunctionService - val apiKey = - context.settingsDataStore.data - .first()[stringPreferencesKey("gemini_api_key")] - ?.takeIf { it.isNotBlank() } - ?: BuildConfig.GEMINI_API_KEY.takeIf { it.isNotBlank() } - if (apiKey.isNullOrBlank()) { - throw IllegalStateException( - "Gemini API key is not configured. Please set gemini_api_key in settings.", - ) - } + val apiKey = getOrFetchApiKey() + val requestPayload = buildImageGenerationPayload(prompt, aspectRatio) + val responseText = executeImageRequest(apiKey, requestPayload) + saveBase64ImageToCache(responseText, prompt) + } - val requestJson = - JSONObject().apply { + private suspend fun getOrFetchApiKey(): String { + val apiKey = + settingsDataStore.data + .first()[stringPreferencesKey("gemini_api_key")] + ?.takeIf { it.isNotBlank() } + ?: BuildConfig.GEMINI_API_KEY.takeIf { it.isNotBlank() } + if (apiKey.isNullOrBlank()) { + throw IllegalStateException( + "Gemini API key is not configured. Please set gemini_api_key in settings.", + ) + } + return apiKey + } + + private fun buildImageGenerationPayload( + prompt: String, + aspectRatio: String?, + ): JSONObject = + JSONObject().apply { + put( + "contents", + JSONArray().apply { put( - "contents", - JSONArray().apply { + JSONObject().apply { put( - JSONObject().apply { - put( - "parts", - JSONArray().apply { - put(JSONObject().apply { put("text", prompt) }) - }, - ) + "parts", + JSONArray().apply { + put(JSONObject().apply { put("text", prompt) }) }, ) }, ) - put( - "generationConfig", - JSONObject().apply { - put("responseModalities", JSONArray().apply { put("IMAGE") }) - if (!aspectRatio.isNullOrBlank()) { - put( - "imageConfig", - JSONObject().apply { - put("aspectRatio", aspectRatio) - }, - ) - } - }, - ) - } + }, + ) + put( + "generationConfig", + JSONObject().apply { + put("responseModalities", JSONArray().apply { put("IMAGE") }) + if (!aspectRatio.isNullOrBlank()) { + put( + "imageConfig", + JSONObject().apply { + put("aspectRatio", aspectRatio) + }, + ) + } + }, + ) + } - val endpointUrl = - "https://generativelanguage.googleapis.com/v1beta/models/gemini-3.1-flash-image-preview:generateContent?key=$apiKey" - val url = URL(endpointUrl) - val connection = - (url.openConnection() as HttpURLConnection).apply { - requestMethod = "POST" - setRequestProperty("Content-Type", "application/json") - doOutput = true - connectTimeout = 30000 - readTimeout = 60000 - } + private fun executeImageRequest( + apiKey: String, + requestJson: JSONObject, + ): String { + val endpointUrl = + "https://generativelanguage.googleapis.com/v1beta/models/gemini-3.1-flash-image-preview:generateContent?key=$apiKey" + val url = URL(endpointUrl) + val connection = + (url.openConnection() as HttpURLConnection).apply { + requestMethod = "POST" + setRequestProperty("Content-Type", "application/json") + doOutput = true + connectTimeout = 30000 + readTimeout = 60000 + } - try { - connection.outputStream.use { os -> - os.write(requestJson.toString().toByteArray(Charsets.UTF_8)) - } + try { + connection.outputStream.use { os -> + os.write(requestJson.toString().toByteArray(Charsets.UTF_8)) + } - val responseCode = connection.responseCode - if (responseCode != HttpURLConnection.HTTP_OK) { - val errorBody = - connection.errorStream?.bufferedReader()?.use { it.readText() } - ?: "HTTP $responseCode" - throw IllegalStateException( - "Image generation failed ($responseCode): $errorBody", - ) - } + val responseCode = connection.responseCode + if (responseCode != HttpURLConnection.HTTP_OK) { + val errorBody = + connection.errorStream?.bufferedReader()?.use { it.readText() } + ?: "HTTP $responseCode" + throw IllegalStateException( + "Image generation failed ($responseCode): $errorBody", + ) + } - val responseText = connection.inputStream.bufferedReader().use { it.readText() } - val responseJson = JSONObject(responseText) - val candidates = responseJson.optJSONArray("candidates") - if (candidates == null || candidates.length() == 0) { - throw IllegalStateException( - "No candidates returned from Gemini image generation API", - ) - } + return connection.inputStream.bufferedReader().use { it.readText() } + } finally { + connection.disconnect() + } + } - val parts = - candidates - .getJSONObject(0) - .optJSONObject("content") - ?.optJSONArray("parts") - if (parts == null || parts.length() == 0) { - throw IllegalStateException( - "No parts returned in candidate content. Gemini response: $responseText", - ) - } + private fun saveBase64ImageToCache( + responseText: String, + prompt: String, + ): GeneratedImageResult { + val responseJson = JSONObject(responseText) + val candidates = responseJson.optJSONArray("candidates") + if (candidates == null || candidates.length() == 0) { + throw IllegalStateException( + "No candidates returned from Gemini image generation API", + ) + } - var base64Data: String? = null - var mimeType = "image/png" - for (i in 0 until parts.length()) { + val parts = + candidates + .getJSONObject(0) + .optJSONObject("content") + ?.optJSONArray("parts") + if (parts == null || parts.length() == 0) { + throw IllegalStateException( + "No parts returned in candidate content. Gemini response: $responseText", + ) + } + + val candidateData = + (0 until parts.length()) + .asSequence() + .mapNotNull { i -> val part = parts.getJSONObject(i) val inlineData = part.optJSONObject("inlineData") ?: part.optJSONObject("inline_data") if (inlineData != null) { - base64Data = inlineData.optString("data") + val base64 = inlineData.optString("data") val returnedMime = inlineData .optString("mimeType") .takeIf { it.isNotBlank() } ?: inlineData.optString("mime_type") - if (returnedMime.isNotBlank()) { - mimeType = returnedMime - } - break + .takeIf { it.isNotBlank() } + ?: "image/png" + base64 to returnedMime + } else { + null } } + .firstOrNull() - if (base64Data.isNullOrBlank()) { - throw IllegalStateException( - "No inlineData image found in response parts. Gemini response: $responseText", - ) - } - - val imageBytes = Base64.decode(base64Data, Base64.DEFAULT) - val extension = - if (mimeType.contains("jpeg") || mimeType.contains("jpg")) "jpg" else "png" - val cachedFile = - File( - context.cacheDir, - "generated_${UUID.randomUUID()}.$extension", - ) - cachedFile.writeBytes(imageBytes) + if (candidateData == null || candidateData.first.isBlank()) { + throw IllegalStateException( + "No inlineData image found in response parts. Gemini response: $responseText", + ) + } - val authority = "${context.packageName}.fileprovider" - val contentUri = FileProvider.getUriForFile(context, authority, cachedFile) - GeneratedImageResult( - imageUri = contentUri.toString(), - mimeType = mimeType, - prompt = prompt, - ) - } finally { - connection.disconnect() + val (base64Data, mimeType) = candidateData + val imageBytes = Base64.decode(base64Data, Base64.DEFAULT) + val extension = + when { + mimeType.contains("jpeg") || mimeType.contains("jpg") -> "jpg" + else -> "png" } - } + val cachedFile = + File( + cacheDir, + "generated_${UUID.randomUUID()}.$extension", + ) + cachedFile.writeBytes(imageBytes) + + val authority = "$packageName.fileprovider" + val contentUri = FileProvider.getUriForFile(this, authority, cachedFile) + return GeneratedImageResult( + imageUri = contentUri.toString(), + mimeType = mimeType, + prompt = prompt, + ) + } /** Represents the result of an image generation request. */ @AppFunctionSerializable From 9be5eefa5669b98c0390e1b812bc1ab7d5116c17 Mon Sep 17 00:00:00 2001 From: Caleb Areeveso Date: Wed, 15 Jul 2026 10:17:53 +0000 Subject: [PATCH 10/16] test: refactor AgentOrchestratorTest expression functions to block bodies and extract mock helpers Change-Id: Id24a66497c47cc87d11cf37fec17fc196a710de9 --- .../agent/domain/AgentOrchestratorTest.kt | 155 ++++++++---------- 1 file changed, 68 insertions(+), 87 deletions(-) diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt index b8a288f..f72670b 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt @@ -395,7 +395,7 @@ class AgentOrchestratorTest { } @Test - fun `observeAndProcessMessages extracts attachments when tool returns imageUri and mimeType`() = + fun `observeAndProcessMessages extracts attachments when tool returns imageUri and mimeType`() { runTest { val threadId = "thread_1" val message = createUserMessage(threadId, "generate image of a dog") @@ -413,51 +413,14 @@ class AgentOrchestratorTest { arguments = mapOf("prompt" to "dog"), callId = "call_1", ) - - val firstResponse = - LlmResponse.Success( - interactionId = "interaction_1", - parts = listOf(toolCall), - ) - val secondResponse = - LlmResponse.Success( - interactionId = "interaction_2", - parts = listOf(LlmResponsePart.Text("Here is your image!")), - ) - - coEvery { - llmProvider.generateResponse( - previousInteractionId = null, - input = any(), - tools = any(), - apiKey = any(), - modelName = any(), - ) - } returns firstResponse - - coEvery { - llmProvider.generateResponse( - previousInteractionId = "interaction_1", - input = any(), - tools = any(), - apiKey = any(), - modelName = any(), - ) - } returns secondResponse - - coEvery { - convertInputToAppFunctionDataUseCase(any(), any(), any()) - } returns Result.success(AppFunctionData.EMPTY) - - coEvery { - executeAppFunctionUseCase(any(), any(), any()) - } returns - ExecuteAppFunctionResult.Data( - data = AppFunctionData.EMPTY, - formattedJson = - """{"imageUri":"content://com.example.appfunctions.agent.fileprovider/cache/test.jpg",""" + - """"mimeType":"image/jpeg","prompt":"dog"}""", - ) + setupTwoStepToolCallAndExecution( + llmProvider = llmProvider, + toolCall = toolCall, + secondResponseText = "Here is your image!", + toolResultJson = + """{"imageUri":"content://com.example.appfunctions.agent.fileprovider/cache/test.jpg",""" + + """"mimeType":"image/jpeg","prompt":"dog"}""", + ) agentOrchestrator.observeAndProcessMessages(threadId) @@ -475,9 +438,10 @@ class AgentOrchestratorTest { ) } } + } @Test - fun `observeAndProcessMessages grants URI read permission when tool is called with content URI argument`() = + fun `observeAndProcessMessages grants URI read permission when tool is called with content URI argument`() { runTest { val threadId = "thread_1" val message = createUserMessage(threadId, "set wallpaper") @@ -496,46 +460,12 @@ class AgentOrchestratorTest { arguments = mapOf("uri" to contentUri), callId = "call_2", ) - - coEvery { - llmProvider.generateResponse( - previousInteractionId = null, - input = any(), - tools = any(), - apiKey = any(), - modelName = any(), - ) - } returns - LlmResponse.Success( - interactionId = "interaction_1", - parts = listOf(toolCall), - ) - - coEvery { - llmProvider.generateResponse( - previousInteractionId = "interaction_1", - input = any(), - tools = any(), - apiKey = any(), - modelName = any(), - ) - } returns - LlmResponse.Success( - interactionId = "interaction_2", - parts = listOf(LlmResponsePart.Text("Wallpaper set")), - ) - - coEvery { - convertInputToAppFunctionDataUseCase(any(), any(), any()) - } returns Result.success(AppFunctionData.EMPTY) - - coEvery { - executeAppFunctionUseCase(any(), any(), any()) - } returns - ExecuteAppFunctionResult.Data( - data = AppFunctionData.EMPTY, - formattedJson = """{"success":true}""", - ) + setupTwoStepToolCallAndExecution( + llmProvider = llmProvider, + toolCall = toolCall, + secondResponseText = "Wallpaper set", + toolResultJson = """{"success":true}""", + ) agentOrchestrator.observeAndProcessMessages(threadId) @@ -547,4 +477,55 @@ class AgentOrchestratorTest { ) } } + } + + private fun setupTwoStepToolCallAndExecution( + llmProvider: LlmProvider, + toolCall: LlmResponsePart.ToolCall, + secondResponseText: String, + toolResultJson: String, + ) { + val firstResponse = + LlmResponse.Success( + interactionId = "interaction_1", + parts = listOf(toolCall), + ) + val secondResponse = + LlmResponse.Success( + interactionId = "interaction_2", + parts = listOf(LlmResponsePart.Text(secondResponseText)), + ) + + coEvery { + llmProvider.generateResponse( + previousInteractionId = null, + input = any(), + tools = any(), + apiKey = any(), + modelName = any(), + ) + } returns firstResponse + + coEvery { + llmProvider.generateResponse( + previousInteractionId = "interaction_1", + input = any(), + tools = any(), + apiKey = any(), + modelName = any(), + ) + } returns secondResponse + + coEvery { + convertInputToAppFunctionDataUseCase(any(), any(), any()) + } returns Result.success(AppFunctionData.EMPTY) + + coEvery { + executeAppFunctionUseCase(any(), any(), any()) + } returns + ExecuteAppFunctionResult.Data( + data = AppFunctionData.EMPTY, + formattedJson = toolResultJson, + ) + } } From 7bac6d4764830643458e1f1e28324317cad14955 Mon Sep 17 00:00:00 2001 From: Caleb Areeveso Date: Mon, 20 Jul 2026 15:09:23 +0000 Subject: [PATCH 11/16] docs: add KDoc documentation to WallpaperRepository Change-Id: I99971c606c783a96e340cc8f947f65d90e71ab73 --- .../example/chatapp/data/WallpaperRepository.kt | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/ChatApp/shared/src/main/kotlin/com/example/chatapp/data/WallpaperRepository.kt b/ChatApp/shared/src/main/kotlin/com/example/chatapp/data/WallpaperRepository.kt index e8a85f6..7509a90 100644 --- a/ChatApp/shared/src/main/kotlin/com/example/chatapp/data/WallpaperRepository.kt +++ b/ChatApp/shared/src/main/kotlin/com/example/chatapp/data/WallpaperRepository.kt @@ -33,9 +33,25 @@ import java.io.InputStream import javax.inject.Inject import javax.inject.Singleton +/** + * Repository for managing custom chat background wallpapers. + */ interface WallpaperRepository { + /** + * Retrieves the wallpaper file path for a specific chat. + * + * @param chatId The ID of the chat. + * @return A [Flow] emitting the wallpaper file path, or `null` if no custom wallpaper is set. + */ fun getWallpaper(chatId: String): Flow + /** + * Saves a wallpaper image for a specific chat. + * + * @param chatId The ID of the chat. + * @param inputStream The input stream of the wallpaper image. + * @return `true` if the wallpaper was saved successfully, `false` otherwise. + */ suspend fun setWallpaper( chatId: String, inputStream: InputStream, From 0d61c7cfa67a4e7ac5c9bdda1362bd72656ecfbe Mon Sep 17 00:00:00 2001 From: Caleb Areeveso Date: Mon, 20 Jul 2026 15:12:46 +0000 Subject: [PATCH 12/16] fix: copy new wallpaper file before deleting old files in WallpaperRepository Change-Id: I2ccd86d2d895b70706477213478ea39cbffa33c6 --- .../com/example/chatapp/data/WallpaperRepository.kt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ChatApp/shared/src/main/kotlin/com/example/chatapp/data/WallpaperRepository.kt b/ChatApp/shared/src/main/kotlin/com/example/chatapp/data/WallpaperRepository.kt index 7509a90..986e343 100644 --- a/ChatApp/shared/src/main/kotlin/com/example/chatapp/data/WallpaperRepository.kt +++ b/ChatApp/shared/src/main/kotlin/com/example/chatapp/data/WallpaperRepository.kt @@ -84,12 +84,13 @@ class WallpaperRepositoryImpl withContext(Dispatchers.IO) { try { val dir = File(context.filesDir, "wallpapers").apply { mkdirs() } - dir.listFiles { f -> f.name.startsWith("wallpaper_${chatId}_") }?.forEach { it.delete() } - val file = File(dir, "wallpaper_${chatId}_${System.currentTimeMillis()}.jpg") - file.outputStream().use { output -> + val newFile = File(dir, "wallpaper_${chatId}_${System.currentTimeMillis()}.jpg") + newFile.outputStream().use { output -> inputStream.copyTo(output) } - wallpapers.update { current -> current + (chatId to file.absolutePath) } + dir.listFiles { f -> f.name.startsWith("wallpaper_${chatId}_") && f != newFile } + ?.forEach { it.delete() } + wallpapers.update { current -> current + (chatId to newFile.absolutePath) } true } catch (e: Exception) { false From cd99d5d7d9163f9bc0714cbae961db61dec5fe47 Mon Sep 17 00:00:00 2001 From: Caleb Areeveso Date: Mon, 20 Jul 2026 19:19:49 +0000 Subject: [PATCH 13/16] fix: deduplicate function declarations in Gemini tool payload Change-Id: I673954d3ebe987a570dadbdca5807e5b09744a90 --- .../agent/data/GeminiProviderImpl.kt | 22 ++++++++++--------- .../agent/domain/AgentOrchestrator.kt | 13 ++++++----- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt index 62438db..f6a2310 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt @@ -58,18 +58,20 @@ class GeminiProviderImpl modelName: String, ): LlmResponse { val convertedTools = - tools.mapNotNull { tool -> - try { - buildJsonObject { - put(KEY_TYPE, JsonPrimitive(VALUE_FUNCTION)) - val functionSchema = toolConverter.convert(tool) - functionSchema.forEach { (key, value) -> put(key, value) } + tools + .distinctBy { toolConverter.getToolName(it) } + .mapNotNull { tool -> + try { + buildJsonObject { + put(KEY_TYPE, JsonPrimitive(VALUE_FUNCTION)) + val functionSchema = toolConverter.convert(tool) + functionSchema.forEach { (key, value) -> put(key, value) } + } + } catch (e: IllegalArgumentException) { + Log.e(TAG, "Failed to convert tool ${tool.id}: ${e.message}", e) + null } - } catch (e: IllegalArgumentException) { - Log.e(TAG, "Failed to convert tool ${tool.id}: ${e.message}", e) - null } - } val requestBody = buildJsonObject { diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt index df9c8d3..b2082a9 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt @@ -152,11 +152,14 @@ class AgentOrchestrator disconnectedApps: Set, targetPackageName: String?, ): List { - return allTools.filter { metadata -> - metadata.isEnabled && - metadata.packageName !in disconnectedApps && - (targetPackageName == null || metadata.packageName == targetPackageName) - } + return allTools + .filter { metadata -> + metadata.isEnabled && + metadata.packageName !in disconnectedApps && + (targetPackageName == null || metadata.packageName == targetPackageName) + } + .sortedByDescending { metadata -> metadata.id.startsWith(metadata.packageName) } + .distinctBy { metadata -> metadata.id } } private suspend fun runInteractionLoop( From e1e273b10fe4e0aef93e609bec75e1184ae87a93 Mon Sep 17 00:00:00 2001 From: Caleb Areeveso Date: Mon, 20 Jul 2026 19:21:30 +0000 Subject: [PATCH 14/16] style: apply spotless Change-Id: I056f58d80290c561500998f3f67ad89042007776 --- agent/app/src/main/res/drawable/ic_launcher_background.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/app/src/main/res/drawable/ic_launcher_background.xml b/agent/app/src/main/res/drawable/ic_launcher_background.xml index 7c323a2..41fdccf 100644 --- a/agent/app/src/main/res/drawable/ic_launcher_background.xml +++ b/agent/app/src/main/res/drawable/ic_launcher_background.xml @@ -34,7 +34,7 @@ - + Date: Tue, 21 Jul 2026 08:13:37 +0000 Subject: [PATCH 15/16] fix: include voiceNoteUri in KNOWN_FILE_REFERENCE_PARAM_NAMES and update test Change-Id: Iba93fd0623d5e07f0ecf0a9f577db98fe2e33d99 --- .../example/appfunctions/agent/domain/AgentOrchestrator.kt | 6 +++++- .../appfunctions/agent/domain/AgentOrchestratorTest.kt | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt index 35d2b59..8b44065 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt @@ -20,6 +20,8 @@ import android.content.Context import android.content.Intent import android.net.Uri import android.util.Log +import androidx.appfunctions.metadata.AppFunctionArrayTypeMetadata +import androidx.appfunctions.metadata.AppFunctionDataTypeMetadata import androidx.appfunctions.metadata.AppFunctionMetadata import androidx.appfunctions.metadata.AppFunctionObjectTypeMetadata import androidx.appfunctions.metadata.AppFunctionParameterMetadata @@ -58,6 +60,9 @@ import kotlinx.coroutines.flow.first import kotlinx.coroutines.flow.stateIn import kotlinx.coroutines.withContext import org.json.JSONObject +import java.io.File +import java.net.HttpURLConnection +import java.net.URL import java.util.UUID import javax.inject.Inject import javax.inject.Singleton @@ -481,7 +486,6 @@ class AgentOrchestrator processingStatus = MessageProcessingStatus.FAILED, ) } -<<<<<<< HEAD private fun resolveRemoteFileReferencesRecursively( context: Context, diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt index bf6ab16..f72670b 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/AgentOrchestratorTest.kt @@ -412,6 +412,7 @@ class AgentOrchestratorTest { functionId = "generateImage", arguments = mapOf("prompt" to "dog"), callId = "call_1", + ) setupTwoStepToolCallAndExecution( llmProvider = llmProvider, toolCall = toolCall, From d2850b470d048e3ef09868dbd6ffb4ed7df0e4ef Mon Sep 17 00:00:00 2001 From: Caleb Areeveso Date: Tue, 21 Jul 2026 08:20:00 +0000 Subject: [PATCH 16/16] style: apply spotless formatting to InputBar.kt Change-Id: Ifeb2b15e846755bbbf351b8da9b617bd4d2f1539 --- ChatApp/app/src/main/kotlin/com/example/chatapp/InputBar.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/ChatApp/app/src/main/kotlin/com/example/chatapp/InputBar.kt b/ChatApp/app/src/main/kotlin/com/example/chatapp/InputBar.kt index 3cc9724..aabad64 100644 --- a/ChatApp/app/src/main/kotlin/com/example/chatapp/InputBar.kt +++ b/ChatApp/app/src/main/kotlin/com/example/chatapp/InputBar.kt @@ -37,7 +37,6 @@ import androidx.compose.material.icons.automirrored.filled.Send import androidx.compose.material.icons.filled.Add import androidx.compose.material3.FilledIconButton import androidx.compose.material3.Icon -import androidx.compose.material3.IconButton import androidx.compose.material3.IconButtonDefaults import androidx.compose.material3.MaterialTheme import androidx.compose.material3.Surface