diff --git a/ChatApp/app/src/main/kotlin/com/example/chatapp/InputBar.kt b/ChatApp/app/src/main/kotlin/com/example/chatapp/InputBar.kt index 2c8c382..aabad64 100644 --- a/ChatApp/app/src/main/kotlin/com/example/chatapp/InputBar.kt +++ b/ChatApp/app/src/main/kotlin/com/example/chatapp/InputBar.kt @@ -37,7 +37,7 @@ import androidx.compose.material.icons.automirrored.filled.Send import androidx.compose.material.icons.filled.Add import androidx.compose.material3.FilledIconButton import androidx.compose.material3.Icon -import androidx.compose.material3.IconButton +import androidx.compose.material3.IconButtonDefaults import androidx.compose.material3.MaterialTheme import androidx.compose.material3.Surface import androidx.compose.material3.Text @@ -77,7 +77,8 @@ internal fun InputBar( Surface( modifier = modifier, - tonalElevation = 3.dp, + color = Color.Transparent, + tonalElevation = 0.dp, ) { Column { if (selectedImages.isNotEmpty()) { @@ -107,18 +108,25 @@ internal fun InputBar( verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(4.dp), ) { - IconButton( + FilledIconButton( onClick = { photoPickerLauncher.launch( PickVisualMediaRequest(ActivityResultContracts.PickVisualMedia.ImageOnly), ) }, + modifier = Modifier.size(56.dp), + colors = + IconButtonDefaults.filledIconButtonColors( + containerColor = MaterialTheme.colorScheme.surfaceVariant, + contentColor = MaterialTheme.colorScheme.onSurfaceVariant, + ), ) { Icon( imageVector = Icons.Default.Add, contentDescription = "Add Image", ) } + Spacer(modifier = Modifier.width(4.dp)) TextField( value = value, onValueChange = onInputChanged, diff --git a/ChatApp/app/src/main/kotlin/com/example/chatapp/uicomponents/ChatScreen.kt b/ChatApp/app/src/main/kotlin/com/example/chatapp/uicomponents/ChatScreen.kt index 13f1b9d..1ba9981 100644 --- a/ChatApp/app/src/main/kotlin/com/example/chatapp/uicomponents/ChatScreen.kt +++ b/ChatApp/app/src/main/kotlin/com/example/chatapp/uicomponents/ChatScreen.kt @@ -15,7 +15,9 @@ */ package com.example.chatapp.uicomponents +import androidx.compose.foundation.background import androidx.compose.foundation.layout.Arrangement +import androidx.compose.foundation.layout.Box import androidx.compose.foundation.layout.Column import androidx.compose.foundation.layout.PaddingValues import androidx.compose.foundation.layout.consumeWindowInsets @@ -57,6 +59,7 @@ import androidx.compose.runtime.setValue import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.draw.clip +import androidx.compose.ui.graphics.Color import androidx.compose.ui.input.nestedscroll.nestedScroll import androidx.compose.ui.layout.ContentScale import androidx.compose.ui.res.stringResource @@ -90,99 +93,123 @@ fun ChatScreen( val uiState by viewModel.uiState.collectAsStateWithLifecycle() var message by rememberSaveable { mutableStateOf("") } - Scaffold( - modifier = - Modifier - .fillMaxSize() - .nestedScroll(scrollBehavior.nestedScrollConnection), - topBar = { - TopAppBar( - colors = - topAppBarColors( - containerColor = MaterialTheme.colorScheme.primaryContainer, - titleContentColor = MaterialTheme.colorScheme.onPrimaryContainer, - ), - title = { - Text(text = viewModel.recipient.name) - }, - navigationIcon = { - IconButton(onClick = onBackClick) { - Icon( - imageVector = Icons.AutoMirrored.Filled.ArrowBack, - contentDescription = "Back", - ) - } - }, - actions = { - IconButton(onClick = onCallClick) { - Icon( - imageVector = Icons.Default.Call, - contentDescription = "Call", - ) - } - }, + Box(modifier = Modifier.fillMaxSize()) { + uiState.wallpaperPath?.let { path -> + AsyncImage( + model = path, + contentDescription = "Chat Wallpaper", + contentScale = ContentScale.Crop, + modifier = Modifier.fillMaxSize(), ) - }, - bottomBar = { - InputBar( - value = message, - placeholder = stringResource(R.string.input_placeholder), - onInputChanged = { - message = it - }, - onSendClick = { uris -> - viewModel.sendMessage(message, uris) - message = "" - }, - sendEnabled = uiState.botMessageState !is BotMessageState.Generating, + Box( modifier = Modifier - .navigationBarsPadding() - .imePadding(), + .fillMaxSize() + .background(Color.Black.copy(alpha = 0.45f)), ) - }, - ) { innerPadding -> - Column( + } + + Scaffold( modifier = Modifier .fillMaxSize() - .padding(innerPadding) - .consumeWindowInsets(innerPadding), - ) { - MessageList( + .nestedScroll(scrollBehavior.nestedScrollConnection), + containerColor = + if (uiState.wallpaperPath != null) { + Color.Transparent + } else { + MaterialTheme.colorScheme.surfaceVariant + }, + topBar = { + TopAppBar( + colors = + topAppBarColors( + containerColor = MaterialTheme.colorScheme.primaryContainer, + titleContentColor = MaterialTheme.colorScheme.onPrimaryContainer, + ), + title = { + Text(text = viewModel.recipient.name) + }, + navigationIcon = { + IconButton(onClick = onBackClick) { + Icon( + imageVector = Icons.AutoMirrored.Filled.ArrowBack, + contentDescription = "Back", + ) + } + }, + actions = { + IconButton(onClick = onCallClick) { + Icon( + imageVector = Icons.Default.Call, + contentDescription = "Call", + ) + } + }, + ) + }, + bottomBar = { + InputBar( + value = message, + placeholder = stringResource(R.string.input_placeholder), + onInputChanged = { + message = it + }, + onSendClick = { uris -> + viewModel.sendMessage(message, uris) + message = "" + }, + sendEnabled = uiState.botMessageState !is BotMessageState.Generating, + modifier = + Modifier + .navigationBarsPadding() + .imePadding(), + ) + }, + ) { innerPadding -> + Box( modifier = Modifier - .fillMaxWidth() - .padding(horizontal = 16.dp) - .weight(1f), - messages = uiState.messages, - contentPadding = PaddingValues(bottom = 8.dp), - ) - - when (val state = uiState.botMessageState) { - is BotMessageState.Generating -> { - CircularProgressIndicator( + .fillMaxSize() + .padding(innerPadding) + .consumeWindowInsets(innerPadding), + ) { + Column(modifier = Modifier.fillMaxSize()) { + MessageList( modifier = Modifier - .padding(vertical = 8.dp) - .align(Alignment.CenterHorizontally), + .fillMaxWidth() + .padding(horizontal = 16.dp) + .weight(1f), + messages = uiState.messages, + contentPadding = PaddingValues(bottom = 8.dp), ) - } - is BotMessageState.Error -> { - AlertDialog( - onDismissRequest = { viewModel.dismissError() }, - title = { Text(text = stringResource(R.string.error)) }, - text = { Text(text = state.errorMessage) }, - confirmButton = { - Button(onClick = { viewModel.dismissError() }) { - Text(text = stringResource(R.string.dismiss_button)) - } - }, - ) - } + when (val state = uiState.botMessageState) { + is BotMessageState.Generating -> { + CircularProgressIndicator( + modifier = + Modifier + .padding(vertical = 8.dp) + .align(Alignment.CenterHorizontally), + ) + } - else -> { // No additional UI for waiting state + is BotMessageState.Error -> { + AlertDialog( + onDismissRequest = { viewModel.dismissError() }, + title = { Text(text = stringResource(R.string.error)) }, + text = { Text(text = state.errorMessage) }, + confirmButton = { + Button(onClick = { viewModel.dismissError() }) { + Text(text = stringResource(R.string.dismiss_button)) + } + }, + ) + } + + else -> {} + } } } } diff --git a/ChatApp/gradle/gradle-daemon-jvm.properties b/ChatApp/gradle/gradle-daemon-jvm.properties new file mode 100644 index 0000000..6c1139e --- /dev/null +++ b/ChatApp/gradle/gradle-daemon-jvm.properties @@ -0,0 +1,12 @@ +#This file is generated by updateDaemonJvm +toolchainUrl.FREE_BSD.AARCH64=https\://api.foojay.io/disco/v3.0/ids/ec7520a1e057cd116f9544c42142a16b/redirect +toolchainUrl.FREE_BSD.X86_64=https\://api.foojay.io/disco/v3.0/ids/4c4f879899012ff0a8b2e2117df03b0e/redirect +toolchainUrl.LINUX.AARCH64=https\://api.foojay.io/disco/v3.0/ids/ec7520a1e057cd116f9544c42142a16b/redirect +toolchainUrl.LINUX.X86_64=https\://api.foojay.io/disco/v3.0/ids/4c4f879899012ff0a8b2e2117df03b0e/redirect +toolchainUrl.MAC_OS.AARCH64=https\://api.foojay.io/disco/v3.0/ids/73bcfb608d1fde9fb62e462f834a3299/redirect +toolchainUrl.MAC_OS.X86_64=https\://api.foojay.io/disco/v3.0/ids/846ee0d876d26a26f37aa1ce8de73224/redirect +toolchainUrl.UNIX.AARCH64=https\://api.foojay.io/disco/v3.0/ids/ec7520a1e057cd116f9544c42142a16b/redirect +toolchainUrl.UNIX.X86_64=https\://api.foojay.io/disco/v3.0/ids/4c4f879899012ff0a8b2e2117df03b0e/redirect +toolchainUrl.WINDOWS.AARCH64=https\://api.foojay.io/disco/v3.0/ids/9482ddec596298c84656d31d16652665/redirect +toolchainUrl.WINDOWS.X86_64=https\://api.foojay.io/disco/v3.0/ids/39701d92e1756bb2f141eb67cd4c660e/redirect +toolchainVersion=21 diff --git a/ChatApp/shared/src/main/kotlin/com/example/chatapp/ChatViewModel.kt b/ChatApp/shared/src/main/kotlin/com/example/chatapp/ChatViewModel.kt index be30019..7524d6f 100644 --- a/ChatApp/shared/src/main/kotlin/com/example/chatapp/ChatViewModel.kt +++ b/ChatApp/shared/src/main/kotlin/com/example/chatapp/ChatViewModel.kt @@ -23,6 +23,7 @@ import com.example.chatapp.data.CallManager import com.example.chatapp.data.DisplayMessage import com.example.chatapp.data.MessageRepository import com.example.chatapp.data.RecipientsRepository +import com.example.chatapp.data.WallpaperRepository import dagger.assisted.Assisted import dagger.assisted.AssistedFactory import dagger.assisted.AssistedInject @@ -64,6 +65,8 @@ data class ChatbotUiState( val messages: List = listOf(), /** The current state of the bot's response generation. */ val botMessageState: BotMessageState = BotMessageState.WaitingForMessage, + /** Optional path to a custom wallpaper image for this chat. */ + val wallpaperPath: String? = null, ) @HiltViewModel(assistedFactory = ChatViewModel.Factory::class) @@ -74,6 +77,7 @@ class ChatViewModel private val messageRepository: MessageRepository, private val callManager: CallManager, private val recipientsRepository: RecipientsRepository, + private val wallpaperRepository: WallpaperRepository, ) : ViewModel() { @AssistedFactory interface Factory { @@ -102,6 +106,11 @@ class ChatViewModel _uiState.update { it.copy(messages = msgs) } } } + viewModelScope.launch { + wallpaperRepository.getWallpaper(recipientId).collect { path -> + _uiState.update { it.copy(wallpaperPath = path) } + } + } } fun startCall() { diff --git a/ChatApp/shared/src/main/kotlin/com/example/chatapp/appfunctions/BaseChatAppFunctionService.kt b/ChatApp/shared/src/main/kotlin/com/example/chatapp/appfunctions/BaseChatAppFunctionService.kt index 19cfa52..cc13cfd 100644 --- a/ChatApp/shared/src/main/kotlin/com/example/chatapp/appfunctions/BaseChatAppFunctionService.kt +++ b/ChatApp/shared/src/main/kotlin/com/example/chatapp/appfunctions/BaseChatAppFunctionService.kt @@ -29,6 +29,7 @@ import androidx.appfunctions.AppFunctionStringValueConstraint import com.example.chatapp.data.CallManager import com.example.chatapp.data.MessageRepository import com.example.chatapp.data.RecipientsRepository +import com.example.chatapp.data.WallpaperRepository import dagger.hilt.android.AndroidEntryPoint import kotlinx.coroutines.CancellationException import javax.inject.Inject @@ -49,6 +50,8 @@ abstract class BaseChatAppFunctionService : AppFunctionService() { @Inject lateinit var callManager: CallManager + @Inject lateinit var wallpaperRepository: WallpaperRepository + /** * Search for message recipients or chat groups by name or email. * Required workflow: Call this before "send" or "makeCall" to obtain a valid endpointValue (unique ID). @@ -172,4 +175,31 @@ abstract class BaseChatAppFunctionService : AppFunctionService() { PendingIntent.FLAG_ONE_SHOT or PendingIntent.FLAG_IMMUTABLE, ) } + + /** + * Updates the wallpaper image for a specific chat conversation. + * + * @param chatId The unique identifier for the recipient or chat group. + * @param wallpaperUri The URI of the image file to set as the chat wallpaper. + */ + @AppFunction(isDescribedByKDoc = true) + suspend fun updateChatWallpaper( + chatId: String, + wallpaperUri: Uri, + ): Boolean { + val resolvedId = + recipientsRepository.getRecipientById(chatId)?.id + ?: recipientsRepository.getGroupById(chatId)?.id + ?: recipientsRepository.searchAny(chatId, maxCount = 1).firstOrNull()?.endpointValue + ?: chatId + val inputStream = + try { + contentResolver.openInputStream(wallpaperUri) + } catch (e: Exception) { + throw AppFunctionInvalidArgumentException("Cannot open wallpaper stream: ${e.message}") + } ?: throw AppFunctionInvalidArgumentException("Cannot open wallpaper stream") + return inputStream.use { stream -> + wallpaperRepository.setWallpaper(resolvedId, stream) + } + } } diff --git a/ChatApp/shared/src/main/kotlin/com/example/chatapp/data/WallpaperRepository.kt b/ChatApp/shared/src/main/kotlin/com/example/chatapp/data/WallpaperRepository.kt new file mode 100644 index 0000000..986e343 --- /dev/null +++ b/ChatApp/shared/src/main/kotlin/com/example/chatapp/data/WallpaperRepository.kt @@ -0,0 +1,106 @@ +/* + * Copyright 2026 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.chatapp.data + +import android.content.Context +import dagger.Binds +import dagger.Module +import dagger.hilt.InstallIn +import dagger.hilt.android.qualifiers.ApplicationContext +import dagger.hilt.components.SingletonComponent +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.flowOn +import kotlinx.coroutines.flow.map +import kotlinx.coroutines.flow.update +import kotlinx.coroutines.withContext +import java.io.File +import java.io.InputStream +import javax.inject.Inject +import javax.inject.Singleton + +/** + * Repository for managing custom chat background wallpapers. + */ +interface WallpaperRepository { + /** + * Retrieves the wallpaper file path for a specific chat. + * + * @param chatId The ID of the chat. + * @return A [Flow] emitting the wallpaper file path, or `null` if no custom wallpaper is set. + */ + fun getWallpaper(chatId: String): Flow + + /** + * Saves a wallpaper image for a specific chat. + * + * @param chatId The ID of the chat. + * @param inputStream The input stream of the wallpaper image. + * @return `true` if the wallpaper was saved successfully, `false` otherwise. + */ + suspend fun setWallpaper( + chatId: String, + inputStream: InputStream, + ): Boolean +} + +@Singleton +class WallpaperRepositoryImpl + @Inject + constructor( + @ApplicationContext private val context: Context, + ) : WallpaperRepository { + private val wallpapers = MutableStateFlow>(emptyMap()) + + override fun getWallpaper(chatId: String): Flow { + return wallpapers.map { map -> + map[chatId] ?: run { + val dir = File(context.filesDir, "wallpapers") + dir.listFiles { f -> f.name.startsWith("wallpaper_${chatId}_") } + ?.maxByOrNull { it.lastModified() } + ?.absolutePath + } + }.flowOn(Dispatchers.IO) + } + + override suspend fun setWallpaper( + chatId: String, + inputStream: InputStream, + ): Boolean = + withContext(Dispatchers.IO) { + try { + val dir = File(context.filesDir, "wallpapers").apply { mkdirs() } + val newFile = File(dir, "wallpaper_${chatId}_${System.currentTimeMillis()}.jpg") + newFile.outputStream().use { output -> + inputStream.copyTo(output) + } + dir.listFiles { f -> f.name.startsWith("wallpaper_${chatId}_") && f != newFile } + ?.forEach { it.delete() } + wallpapers.update { current -> current + (chatId to newFile.absolutePath) } + true + } catch (e: Exception) { + false + } + } + } + +@Module +@InstallIn(SingletonComponent::class) +abstract class WallpaperModule { + @Binds + abstract fun bindWallpaperRepository(impl: WallpaperRepositoryImpl): WallpaperRepository +} diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt index f6a2310..9ae0864 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiProviderImpl.kt @@ -59,6 +59,7 @@ class GeminiProviderImpl ): LlmResponse { val convertedTools = tools + .sortedByDescending { it.id.startsWith(it.packageName) } .distinctBy { toolConverter.getToolName(it) } .mapNotNull { tool -> try { diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt index 7de144e..cc10981 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/data/GeminiToolConverter.kt @@ -67,12 +67,16 @@ class GeminiToolConverter parameter.dataType, tool.components, tool.id, + parameterName = parameter.name, ) put( parameter.name, buildJsonObject { typeSchema.forEach { (key, value) -> put(key, value) } - put(KEY_DESCRIPTION, JsonPrimitive(parameter.description)) + put( + KEY_DESCRIPTION, + JsonPrimitive(parameter.description), + ) }, ) } @@ -82,7 +86,13 @@ class GeminiToolConverter if (requiredParams.isNotEmpty()) { put( KEY_REQUIRED, - buildJsonArray { requiredParams.forEach { add(JsonPrimitive(it)) } }, + buildJsonArray { + requiredParams.forEach { + add( + JsonPrimitive(it), + ) + } + }, ) } }, @@ -118,11 +128,15 @@ class GeminiToolConverter components: AppFunctionComponentsMetadata, functionId: String, visitedReferences: Set = emptySet(), + parameterName: String? = null, ): JsonObject { return when (dataType) { is AppFunctionStringTypeMetadata -> buildJsonObject { put(KEY_TYPE, JsonPrimitive(VALUE_STRING)) + if (isFileReferenceParameter(parameterName)) { + put(KEY_FORMAT, JsonPrimitive(VALUE_FILE_REFERENCE)) + } val enumValues = dataType.enumValues if (!enumValues.isNullOrEmpty()) { put( @@ -168,39 +182,58 @@ class GeminiToolConverter components, functionId, visitedReferences, + parameterName, ), ) } is AppFunctionObjectTypeMetadata -> - buildJsonObject { - put(KEY_TYPE, JsonPrimitive(VALUE_OBJECT)) - put( - KEY_PROPERTIES, - buildJsonObject { - dataType.properties.forEach { (name, type) -> - put( - name, - mapDataTypeToGeminiSchema( - type, - components, - functionId, - visitedReferences, - ), - ) - } - }, - ) - if (dataType.required.isNotEmpty()) { + if (dataType.qualifiedName == "android.net.Uri") { + buildJsonObject { + put(KEY_TYPE, JsonPrimitive(VALUE_STRING)) + put(KEY_FORMAT, JsonPrimitive(VALUE_FILE_REFERENCE)) + } + } else { + buildJsonObject { + put(KEY_TYPE, JsonPrimitive(VALUE_OBJECT)) put( - KEY_REQUIRED, - buildJsonArray { - dataType.required.forEach { name -> add(JsonPrimitive(name)) } + KEY_PROPERTIES, + buildJsonObject { + dataType.properties.forEach { (name, type) -> + put( + name, + mapDataTypeToGeminiSchema( + type, + components, + functionId, + visitedReferences, + parameterName = name, + ), + ) + } }, ) + if (dataType.required.isNotEmpty()) { + put( + KEY_REQUIRED, + buildJsonArray { + dataType.required.forEach { name -> + add( + JsonPrimitive(name), + ) + } + }, + ) + } } } is AppFunctionReferenceTypeMetadata -> { val referenceKey = dataType.referenceDataType + if (referenceKey == "android.net.Uri") { + return buildJsonObject { + put(KEY_TYPE, JsonPrimitive(VALUE_STRING)) + put(KEY_FORMAT, JsonPrimitive(VALUE_FILE_REFERENCE)) + } + } if (visitedReferences.contains(referenceKey)) { Log.d( "GeminiToolConverter", @@ -218,6 +251,7 @@ class GeminiToolConverter components, functionId, visitedReferences + referenceKey, + parameterName, ) } else -> @@ -227,12 +261,30 @@ class GeminiToolConverter } } + private fun isFileReferenceParameter(parameterName: String?): Boolean { + if (parameterName == null) return false + if (parameterName in KNOWN_FILE_REFERENCE_PARAM_NAMES) return true + return parameterName.endsWith("Uri", ignoreCase = true) || + parameterName.endsWith("Uris", ignoreCase = true) + } + companion object { private const val TOOL_ID_SEPARATOR = "_" private const val KEY_NAME = "name" private const val KEY_DESCRIPTION = "description" private const val KEY_PARAMETERS = "parameters" private const val KEY_TYPE = "type" + private const val KEY_FORMAT = "format" + private const val VALUE_FILE_REFERENCE = "file_reference" + private val KNOWN_FILE_REFERENCE_PARAM_NAMES = + setOf( + "wallpaperUri", + "imageUri", + "attachmentUri", + "ringtoneUri", + "profilePictureUri", + "audioUri", + ) private const val VALUE_OBJECT = "object" private const val KEY_PROPERTIES = "properties" private const val KEY_REQUIRED = "required" diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt index b2082a9..95fcdd1 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/AgentOrchestrator.kt @@ -20,7 +20,13 @@ import android.content.Context import android.content.Intent import android.net.Uri import android.util.Log +import androidx.appfunctions.metadata.AppFunctionArrayTypeMetadata +import androidx.appfunctions.metadata.AppFunctionDataTypeMetadata import androidx.appfunctions.metadata.AppFunctionMetadata +import androidx.appfunctions.metadata.AppFunctionObjectTypeMetadata +import androidx.appfunctions.metadata.AppFunctionParameterMetadata +import androidx.appfunctions.metadata.AppFunctionReferenceTypeMetadata +import androidx.core.content.FileProvider import com.example.appfunctions.agent.data.LlmProviderName import com.example.appfunctions.agent.data.SettingsRepository import com.example.appfunctions.agent.data.db.entities.MessageAttachment @@ -54,6 +60,9 @@ import kotlinx.coroutines.flow.first import kotlinx.coroutines.flow.stateIn import kotlinx.coroutines.withContext import org.json.JSONObject +import java.io.File +import java.net.HttpURLConnection +import java.net.URL import java.util.UUID import javax.inject.Inject import javax.inject.Singleton @@ -366,7 +375,26 @@ class AgentOrchestrator return ExecuteToolCallsResult.Error } - val convertedInputs = toolCall.arguments.filterValues { it != null } as Map + val rawConvertedInputs = toolCall.arguments.filterValues { it != null } as Map + val convertedInputs = + try { + withContext(Dispatchers.IO) { + resolveRemoteFileReferencesRecursively( + context = context, + parametersMetadata = matchingTool.parameters, + inputs = rawConvertedInputs, + ) + } + } catch (e: Exception) { + completeMessageWithError( + message.messageId, + message.threadId, + "Failed to download remote file reference: ${e.message}", + ) + return ExecuteToolCallsResult.Error + } + + grantContentUriPermissionsRecursively(context, toolCall.packageName, convertedInputs) for (value in convertedInputs.values) { if (value is String && value.startsWith("content://")) { @@ -389,7 +417,6 @@ class AgentOrchestrator inputs = convertedInputs, ) } - if (appFunctionDataResult.isFailure) { completeMessageWithError( message.messageId, @@ -472,4 +499,163 @@ class AgentOrchestrator processingStatus = MessageProcessingStatus.FAILED, ) } + + private fun resolveRemoteFileReferencesRecursively( + context: Context, + parametersMetadata: List, + inputs: Map, + ): Map { + val paramMap = parametersMetadata.associateBy { it.name } + return inputs.mapValues { (key, value) -> + val paramMeta = paramMap[key] + resolveValueRecursively(context, paramMeta?.dataType, value, key) + } + } + + private fun resolveValueRecursively( + context: Context, + dataType: AppFunctionDataTypeMetadata?, + value: Any, + paramName: String?, + ): Any { + return when (value) { + is String -> { + val shouldResolve = + isFileReferenceParameter(paramName) || isUriMetadata(dataType) + if (shouldResolve && ( + value.startsWith( + "http://", + ) || value.startsWith("https://") + ) + ) { + downloadRemoteFileToContentUri(context, value) + } else { + value + } + } + is Map<*, *> -> { + value.entries.associate { entry -> + val k = entry.key as String + val propType = + (dataType as? AppFunctionObjectTypeMetadata)?.properties?.get(k) + k to ( + entry.value?.let { + resolveValueRecursively( + context, + propType, + it, + k, + ) + } ?: "" + ) + } + } + is List<*> -> { + val itemType = (dataType as? AppFunctionArrayTypeMetadata)?.itemType + value.mapNotNull { item -> + if (item != null) resolveValueRecursively(context, itemType, item, paramName) else null + } + } + else -> value + } + } + + private fun downloadRemoteFileToContentUri( + context: Context, + urlString: String, + ): String { + val url = URL(urlString) + val connection = + (url.openConnection() as HttpURLConnection).apply { + connectTimeout = 10000 + readTimeout = 15000 + } + try { + connection.connect() + val contentType = connection.contentType ?: "" + val ext = + when { + contentType.contains("png", ignoreCase = true) -> "png" + contentType.contains("jpeg", ignoreCase = true) || + contentType.contains("jpg", ignoreCase = true) -> "jpg" + contentType.contains("gif", ignoreCase = true) -> "gif" + contentType.contains("webp", ignoreCase = true) -> "webp" + urlString.substringAfterLast("/", "").contains(".") -> + urlString.substringAfterLast("/").substringAfterLast(".") + else -> "jpg" + } + val cacheDir = File(context.cacheDir, "file_references").apply { mkdirs() } + val file = File(cacheDir, "generated_${UUID.randomUUID()}.$ext") + connection.inputStream.use { input -> + file.outputStream().use { output -> + input.copyTo(output) + } + } + val contentUri = + FileProvider.getUriForFile( + context, + "${context.packageName}.fileprovider", + file, + ) + return contentUri.toString() + } finally { + connection.disconnect() + } + } + + private fun grantContentUriPermissionsRecursively( + context: Context, + targetPackageName: String, + value: Any?, + ) { + when (value) { + is String -> { + if (value.startsWith("content://")) { + runCatching { + val uri = Uri.parse(value) + context.grantUriPermission( + targetPackageName, + uri, + Intent.FLAG_GRANT_READ_URI_PERMISSION, + ) + } + } + } + is Map<*, *> -> + value.values.forEach { + grantContentUriPermissionsRecursively(context, targetPackageName, it) + } + is List<*> -> + value.forEach { + grantContentUriPermissionsRecursively(context, targetPackageName, it) + } + } + } + + private fun isFileReferenceParameter(parameterName: String?): Boolean { + if (parameterName == null) return false + if (parameterName in KNOWN_FILE_REFERENCE_PARAM_NAMES) return true + return parameterName.endsWith("Uri", ignoreCase = true) || + parameterName.endsWith("Uris", ignoreCase = true) + } + + private fun isUriMetadata(dataType: AppFunctionDataTypeMetadata?): Boolean { + if (dataType == null) return false + if (dataType is AppFunctionObjectTypeMetadata && dataType.qualifiedName == "android.net.Uri") return true + if (dataType is AppFunctionReferenceTypeMetadata && dataType.referenceDataType == "android.net.Uri") return true + return false + } + + companion object { + private val KNOWN_FILE_REFERENCE_PARAM_NAMES = + setOf( + "wallpaperUri", + "imageUri", + "attachmentUri", + "ringtoneUri", + "profilePictureUri", + "audioUri", + "voiceNoteUri", + ) + } } diff --git a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCase.kt b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCase.kt index 7fa27e7..799331a 100644 --- a/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCase.kt +++ b/agent/app/src/main/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCase.kt @@ -98,8 +98,17 @@ class ConvertInputToAppFunctionDataUseCase } } is AppFunctionObjectTypeMetadata -> { - val objData = convertObject(dataType, value as Map, components) - builder.setAppFunctionData(name, objData) + if (dataType.qualifiedName == "android.net.Uri" && value is String) { + val uriPropertyName = dataType.properties.keys.firstOrNull() ?: "uri" + val uriData = + AppFunctionData.Builder(dataType, components) + .setString(uriPropertyName, value) + .build() + builder.setAppFunctionData(name, uriData) + } else { + val objData = convertObject(dataType, value as Map, components) + builder.setAppFunctionData(name, objData) + } } is AppFunctionArrayTypeMetadata -> { setArrayValue(builder, name, dataType, value as List, components) @@ -109,8 +118,17 @@ class ConvertInputToAppFunctionDataUseCase val objectType = components.dataTypes[referenceKey] as? AppFunctionObjectTypeMetadata if (objectType != null) { - val objData = convertObject(objectType, value as Map, components) - builder.setAppFunctionData(name, objData) + if (referenceKey == "android.net.Uri" && value is String) { + val uriPropertyName = objectType.properties.keys.firstOrNull() ?: "uri" + val uriData = + AppFunctionData.Builder(objectType, components) + .setString(uriPropertyName, value) + .build() + builder.setAppFunctionData(name, uriData) + } else { + val objData = convertObject(objectType, value as Map, components) + builder.setAppFunctionData(name, objData) + } } } } diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiToolConverterTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiToolConverterTest.kt index 2816960..62cb16a 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiToolConverterTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/data/GeminiToolConverterTest.kt @@ -408,4 +408,116 @@ class GeminiToolConverterTest { assertEquals(expectedJson, schema) } + + @Test + fun convert_stringParameterEndingWithUri_injectsFileReferenceFormat() { + val parameter = + AppFunctionParameterMetadata( + name = "wallpaperUri", + isRequired = true, + dataType = AppFunctionStringTypeMetadata(isNullable = false), + description = "A URI parameter", + ) + val tool = + AppFunctionMetadata( + id = "com.example.my_function", + packageName = "com.example", + isEnabled = true, + schema = null, + parameters = listOf(parameter), + response = + AppFunctionResponseMetadata( + valueType = AppFunctionStringTypeMetadata(isNullable = false), + description = "", + ), + components = AppFunctionComponentsMetadata(emptyMap()), + description = "Test function", + deprecation = null, + ) + + val schema = converter.convert(tool) + + val expectedJson = + Json.parseToJsonElement( + """ + { + "name": "com_example_my_function", + "description": "Test function", + "parameters": { + "type": "object", + "properties": { + "wallpaperUri": { + "type": "string", + "format": "file_reference", + "description": "A URI parameter" + } + }, + "required": ["wallpaperUri"] + } + } + """, + ) + + assertEquals(expectedJson, schema) + } + + @Test + fun convert_uriObjectType_injectsFileReferenceFormat() { + val uriObjectMetadata = + AppFunctionObjectTypeMetadata( + properties = emptyMap(), + required = emptyList(), + qualifiedName = "android.net.Uri", + isNullable = false, + description = "Uri object", + ) + val parameter = + AppFunctionParameterMetadata( + name = "customUri", + isRequired = true, + dataType = uriObjectMetadata, + description = "A Uri object parameter", + ) + val tool = + AppFunctionMetadata( + id = "com.example.my_function", + packageName = "com.example", + isEnabled = true, + schema = null, + parameters = listOf(parameter), + response = + AppFunctionResponseMetadata( + valueType = AppFunctionStringTypeMetadata(isNullable = false), + description = "", + ), + components = AppFunctionComponentsMetadata(emptyMap()), + description = "Test function", + deprecation = null, + ) + + val schema = converter.convert(tool) + + val expectedJson = + Json.parseToJsonElement( + """ + { + "name": "com_example_my_function", + "description": "Test function", + "parameters": { + "type": "object", + "properties": { + "customUri": { + "type": "string", + "format": "file_reference", + "description": "A Uri object parameter" + } + }, + "required": ["customUri"] + } + } + """, + ) + + assertEquals(expectedJson, schema) + } } diff --git a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCaseTest.kt b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCaseTest.kt index 25760fb..478030b 100644 --- a/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCaseTest.kt +++ b/agent/app/src/test/java/com/example/appfunctions/agent/domain/appfunction/ConvertInputToAppFunctionDataUseCaseTest.kt @@ -238,4 +238,32 @@ class ConvertInputToAppFunctionDataUseCaseTest { assertEquals(true, result.isFailure) } + + @Test + fun convert_uriObjectTypeWithStringInput_buildsUriAppFunctionData() { + val uriObjectType = + AppFunctionObjectTypeMetadata( + properties = + mapOf( + "uri" to AppFunctionStringTypeMetadata(false), + ), + required = listOf("uri"), + qualifiedName = "android.net.Uri", + isNullable = false, + ) + val parameters = + listOf( + AppFunctionParameterMetadata( + name = "wallpaperUri", + isRequired = true, + dataType = uriObjectType, + ), + ) + val inputs = mapOf("wallpaperUri" to "content://com.example/file.jpg") + + val result = useCase(parameters, components, inputs).getOrThrow() + + val uriData = result.getAppFunctionData("wallpaperUri") + assertEquals("content://com.example/file.jpg", uriData?.getString("uri")) + } }