From efe479f4acfcb911c17fb4ce4285f7a3a2b9a1dd Mon Sep 17 00:00:00 2001 From: John Trujillo Date: Wed, 6 May 2026 12:36:08 -0500 Subject: [PATCH 1/2] fix: resolve bounds error in attribute parser and correct tag filtering Added bounds validation in FuzzyAttributeParser and updated UI element filter logic. --- .../domain/FuzzyAttributeParser.kt | 2 + .../domain/YoloToXmlConverter.kt | 111 ++++++++++++------ .../computervision/utils/PlaceholderUtils.kt | 14 +++ 3 files changed, 90 insertions(+), 37 deletions(-) diff --git a/cv-image-to-xml/src/main/java/org/appdevforall/codeonthego/computervision/domain/FuzzyAttributeParser.kt b/cv-image-to-xml/src/main/java/org/appdevforall/codeonthego/computervision/domain/FuzzyAttributeParser.kt index 989f4c9bb7..800b0ed67c 100644 --- a/cv-image-to-xml/src/main/java/org/appdevforall/codeonthego/computervision/domain/FuzzyAttributeParser.kt +++ b/cv-image-to-xml/src/main/java/org/appdevforall/codeonthego/computervision/domain/FuzzyAttributeParser.kt @@ -309,6 +309,8 @@ object FuzzyAttributeParser { annotation.length } + if (current.valueStart > valueEnd) continue + var rawValue = annotation.substring(current.valueStart, valueEnd).trim() if (rawValue.isNotEmpty()) { diff --git a/cv-image-to-xml/src/main/java/org/appdevforall/codeonthego/computervision/domain/YoloToXmlConverter.kt b/cv-image-to-xml/src/main/java/org/appdevforall/codeonthego/computervision/domain/YoloToXmlConverter.kt index 7eff47b579..3eb3f64244 100644 --- a/cv-image-to-xml/src/main/java/org/appdevforall/codeonthego/computervision/domain/YoloToXmlConverter.kt +++ b/cv-image-to-xml/src/main/java/org/appdevforall/codeonthego/computervision/domain/YoloToXmlConverter.kt @@ -4,7 +4,7 @@ import org.appdevforall.codeonthego.computervision.domain.model.DetectionResult import org.appdevforall.codeonthego.computervision.domain.model.ScaledBox import org.appdevforall.codeonthego.computervision.domain.xml.AndroidXmlGenerator import org.appdevforall.codeonthego.computervision.utils.MetadataDetector -import org.appdevforall.codeonthego.computervision.utils.getSortedScaledPlaceholders +import org.appdevforall.codeonthego.computervision.utils.buildPlaceholderOverrides import kotlin.comparisons.compareBy class YoloToXmlConverter( @@ -23,41 +23,31 @@ class YoloToXmlConverter( targetDpHeight: Int, wrapInScroll: Boolean = true ): Pair { - val uiCandidates = detections - .filter { (it.isYolo || it.label == "text") && it.label != "widget_tag" } - .filterNot { MetadataDetector.isMetadataDetection(it.label, it.text) } - .distinctBy { - if (it.label.startsWith("switch")) { - "${((it.boundingBox.top + it.boundingBox.bottom) / 2f).toInt() / 50}" - } else { - "${it.label}:${it.boundingBox.left}:${it.boundingBox.top}:${it.boundingBox.right}:${it.boundingBox.bottom}" - } - } + // 1. Filter and prepare base UI candidates + val uiCandidates = extractUiCandidates(detections) - var scaledBoxes = uiCandidates.map { geometryProcessor.scaleDetection(it, sourceImageWidth, sourceImageHeight, targetDpWidth, targetDpHeight) } + // 2. Scale detections to target DP dimensions + val scaledBoxes = scaleDetections(uiCandidates, sourceImageWidth, sourceImageHeight, targetDpWidth, targetDpHeight) - val parents = scaledBoxes.filter { it.label != "text" && !annotationMatcher.isTag(it.text) } - var texts = scaledBoxes.filter { it.label == "text" && !annotationMatcher.isTag(it.text) } + // 3. Associate isolated text detections to their respective UI widgets + val associatedBoxes = associateTextToWidgets(scaledBoxes) - scaledBoxes = geometryProcessor.assignTextToParents(parents, texts, scaledBoxes) - texts = scaledBoxes.filter { it.label == "text" && !annotationMatcher.isTag(it.text) } - scaledBoxes = geometryProcessor.assignNearbyTextToWidgets(scaledBoxes, texts) + // 4. Clean up and finalize the UI elements list + val uiElements = finalizeUiElements(associatedBoxes) - val uiElements = scaledBoxes.filter { - !annotationMatcher.isTag(it.text) && !MetadataDetector.isMetadataDetection(it.label, it.text) - } - val widgetTags = detections.filter { it.label == "widget_tag" || (!it.isYolo && annotationMatcher.isTag(it.text)) } - val canvasTags = widgetTags.map { geometryProcessor.scaleDetection(it, sourceImageWidth, sourceImageHeight, targetDpWidth, targetDpHeight) } + // 5. Extract and scale reference tags (e.g., T-1, B-1) from the canvas + val canvasTags = extractCanvasTags(detections, sourceImageWidth, sourceImageHeight, targetDpWidth, targetDpHeight) + // 6. Match margin annotations with the extracted UI elements val finalAnnotations = annotationMatcher.matchAnnotationsToElements(canvasTags, uiElements, annotations) + // 7. Sort boxes top-to-bottom, left-to-right for sequential XML rendering val sortedBoxes = uiElements.sortedWith(compareBy({ it.y }, { it.x })) - val selectedImageOverrides = buildSelectedImageOverrides( - uiElements = uiElements, - selectedImagesByPlaceholderId = selectedImagesByPlaceholderId - ) + // 8. Prepare local drawable resources overrides for image placeholders + val selectedImageOverrides = uiElements.buildPlaceholderOverrides(selectedImagesByPlaceholderId) + // 9. Generate final XML output return xmlGenerator.buildXml( boxes = sortedBoxes, annotations = finalAnnotations, @@ -67,17 +57,64 @@ class YoloToXmlConverter( ) } - private fun buildSelectedImageOverrides( - uiElements: List, - selectedImagesByPlaceholderId: Map - ): Map { - val placeholders = uiElements.getSortedScaledPlaceholders() - - return placeholders.mapIndexedNotNull { index, box -> - val drawableReference = selectedImagesByPlaceholderId["ph_$index"] - ?: return@mapIndexedNotNull null - box to drawableReference - }.toMap() + private fun extractUiCandidates(detections: List): List { + return detections + .filter { (it.isYolo || it.label == "text") && it.label != "widget_tag" } + .filterNot { MetadataDetector.isMetadataDetection(it.label, it.text) } + .distinctBy { + if (it.label.startsWith("switch")) { + // Deduplicate switches by grouping them within a 50px vertical band + "${((it.boundingBox.top + it.boundingBox.bottom) / 2f).toInt() / 50}" + } else { + // Exact coordinate deduplication for other widgets + "${it.label}:${it.boundingBox.left}:${it.boundingBox.top}:${it.boundingBox.right}:${it.boundingBox.bottom}" + } + } + } + + private fun scaleDetections( + candidates: List, + sourceWidth: Int, + sourceHeight: Int, + targetWidth: Int, + targetHeight: Int + ): List { + return candidates.map { + geometryProcessor.scaleDetection(it, sourceWidth, sourceHeight, targetWidth, targetHeight) + } + } + + private fun associateTextToWidgets(scaledBoxes: List): List { + val parents = scaledBoxes.filter { it.label != "text" && !annotationMatcher.isTag(it.text) } + val initialTexts = scaledBoxes.filter { it.label == "text" && !annotationMatcher.isTag(it.text) } + + val textAssignedBoxes = geometryProcessor.assignTextToParents(parents, initialTexts, scaledBoxes) + + val remainingTexts = textAssignedBoxes.filter { it.label == "text" && !annotationMatcher.isTag(it.text) } + return geometryProcessor.assignNearbyTextToWidgets(textAssignedBoxes, remainingTexts) + } + + private fun finalizeUiElements(boxes: List): List { + return boxes.filter { + // Keep the widget if it's not pure text, or if it is text but not recognized as a tag. + (it.label != "text" || !annotationMatcher.isTag(it.text)) && + !MetadataDetector.isMetadataDetection(it.label, it.text) + } + } + + private fun extractCanvasTags( + detections: List, + sourceWidth: Int, + sourceHeight: Int, + targetWidth: Int, + targetHeight: Int + ): List { + val widgetTags = detections.filter { + it.label == "widget_tag" || (!it.isYolo && annotationMatcher.isTag(it.text)) + } + return widgetTags.map { + geometryProcessor.scaleDetection(it, sourceWidth, sourceHeight, targetWidth, targetHeight) + } } companion object { diff --git a/cv-image-to-xml/src/main/java/org/appdevforall/codeonthego/computervision/utils/PlaceholderUtils.kt b/cv-image-to-xml/src/main/java/org/appdevforall/codeonthego/computervision/utils/PlaceholderUtils.kt index 2f0d18949b..1a2eff8e4d 100644 --- a/cv-image-to-xml/src/main/java/org/appdevforall/codeonthego/computervision/utils/PlaceholderUtils.kt +++ b/cv-image-to-xml/src/main/java/org/appdevforall/codeonthego/computervision/utils/PlaceholderUtils.kt @@ -14,3 +14,17 @@ fun List.getSortedScaledPlaceholders(): List { return this.filter { it.label == IMAGE_PLACEHOLDER_LABEL } .sortedWith(compareBy({ it.y }, { it.x })) } + +/** + * Associates ordered image placeholders with their selected drawable references. + * Useful for mapping user-selected gallery images to the physical canvas bounding boxes. + */ +fun List.buildPlaceholderOverrides(selectedImagesByPlaceholderId: Map): Map { + val placeholders = this.getSortedScaledPlaceholders() + + return placeholders.mapIndexedNotNull { index, box -> + val drawableReference = selectedImagesByPlaceholderId["ph_$index"] + ?: return@mapIndexedNotNull null + box to drawableReference + }.toMap() +} From a8c257616403dec90ad9659646390200717f81ca Mon Sep 17 00:00:00 2001 From: John Trujillo Date: Thu, 7 May 2026 08:24:55 -0500 Subject: [PATCH 2/2] fix: restrict tag filtering to text boxes on `associateTextToWidgets` method --- .../codeonthego/computervision/domain/YoloToXmlConverter.kt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cv-image-to-xml/src/main/java/org/appdevforall/codeonthego/computervision/domain/YoloToXmlConverter.kt b/cv-image-to-xml/src/main/java/org/appdevforall/codeonthego/computervision/domain/YoloToXmlConverter.kt index 3eb3f64244..206c974c61 100644 --- a/cv-image-to-xml/src/main/java/org/appdevforall/codeonthego/computervision/domain/YoloToXmlConverter.kt +++ b/cv-image-to-xml/src/main/java/org/appdevforall/codeonthego/computervision/domain/YoloToXmlConverter.kt @@ -85,12 +85,11 @@ class YoloToXmlConverter( } private fun associateTextToWidgets(scaledBoxes: List): List { - val parents = scaledBoxes.filter { it.label != "text" && !annotationMatcher.isTag(it.text) } + val parents = scaledBoxes.filter { it.label != "text" } val initialTexts = scaledBoxes.filter { it.label == "text" && !annotationMatcher.isTag(it.text) } - val textAssignedBoxes = geometryProcessor.assignTextToParents(parents, initialTexts, scaledBoxes) - val remainingTexts = textAssignedBoxes.filter { it.label == "text" && !annotationMatcher.isTag(it.text) } + return geometryProcessor.assignNearbyTextToWidgets(textAssignedBoxes, remainingTexts) }