From 58df5f771a6ff429ba4107d11f6df7eb32cac0eb Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Thu, 7 May 2026 12:42:55 +0000 Subject: [PATCH 1/3] feat: implement intent-driven object detection and iterative zoom - Fix user query passthrough in resolution search - Add server-side image cropping utility using sharp - Implement multi-pass iterative analysis loop in resolutionSearch agent - Enhance GeoJSON schema with category and confidence fields - Implement coordinate mapping from zoomed crops to absolute Lat/Lng - Provide incremental UI feedback for summary and Mapbox layers during passes Co-authored-by: ngoiyaeric <115367894+ngoiyaeric@users.noreply.github.com> --- app/actions.tsx | 40 ++++-- bun.lock | 1 + components/chat.tsx | 4 +- components/followup-panel.tsx | 4 + components/header-search-button.tsx | 13 +- lib/agents/resolution-search.tsx | 214 ++++++++++++++++++++++++---- lib/utils/image-server-utils.ts | 51 +++++++ package.json | 1 + 8 files changed, 284 insertions(+), 44 deletions(-) create mode 100644 lib/utils/image-server-utils.ts diff --git a/app/actions.tsx b/app/actions.tsx index 50e985bf..2d37119f 100644 --- a/app/actions.tsx +++ b/app/actions.tsx @@ -57,7 +57,14 @@ async function submit(formData?: FormData, skip?: boolean) { const timezone = (formData?.get('timezone') as string) || 'UTC'; const lat = formData?.get('latitude') ? parseFloat(formData.get('latitude') as string) : undefined; const lng = formData?.get('longitude') ? parseFloat(formData.get('longitude') as string) : undefined; - const location = (lat !== undefined && lng !== undefined) ? { lat, lng } : undefined; + const boundsString = formData?.get('bounds') as string; + let bounds = undefined; + try { + bounds = boundsString ? JSON.parse(boundsString) : undefined; + } catch (e) { + console.error('Failed to parse bounds:', e); + } + const location = (lat !== undefined && lng !== undefined) ? { lat, lng, bounds } : undefined; if (!file) { throw new Error('No file provided for resolution search.'); @@ -81,7 +88,7 @@ async function submit(formData?: FormData, skip?: boolean) { message.type !== 'resolution_search_result' ); - const userInput = 'Analyze this map view.'; + const userInput = (formData?.get('input') as string) || 'Analyze this map view.'; const content: CoreMessage['content'] = [ { type: 'text', text: userInput }, { type: 'image', image: dataUrl, mimeType: file.type } @@ -96,32 +103,43 @@ async function submit(formData?: FormData, skip?: boolean) { }); messages.push({ role: 'user', content }); - const summaryStream = createStreamableValue('Analyzing map view...'); + const summaryStream = createStreamableValue(`Analyzing: ${userInput}`); const groupeId = nanoid(); async function processResolutionSearch() { try { const streamResult = await resolutionSearch(messages, timezone, drawnFeatures, location); + uiStream.append( + + ); + let fullSummary = ''; for await (const partialObject of streamResult.partialObjectStream) { if (partialObject.summary) { fullSummary = partialObject.summary; summaryStream.update(fullSummary); + if (fullSummary.includes('ZOOM PASSContext') || fullSummary.includes('zoomed-in CROP')) { + summaryStream.update(`${fullSummary}\n\n[Zoom Pass in progress...]`); + } + } + if (partialObject.geoJson) { + uiStream.update( + + ); } } const analysisResult = await streamResult.object; summaryStream.done(analysisResult.summary || 'Analysis complete.'); - if (analysisResult.geoJson) { - uiStream.append( - - ); - } + messages.push({ role: 'assistant', content: analysisResult.summary || 'Analysis complete.' }); diff --git a/bun.lock b/bun.lock index f101e5d7..d1b2db7e 100644 --- a/bun.lock +++ b/bun.lock @@ -78,6 +78,7 @@ "rehype-katex": "^7.0.1", "remark-gfm": "^4.0.1", "remark-math": "^6.0.0", + "sharp": "^0.34.5", "sonner": "^1.7.4", "tailwind-merge": "^2.6.0", "tailwindcss-animate": "^1.0.7", diff --git a/components/chat.tsx b/components/chat.tsx index d3560a77..09205114 100644 --- a/components/chat.tsx +++ b/components/chat.tsx @@ -138,7 +138,7 @@ export function Chat({ id }: ChatProps) { if (isMobile) { return ( {/* Add Provider */} - +
{activeView ? : isUsageOpen ? : } @@ -184,7 +184,7 @@ export function Chat({ id }: ChatProps) { // Desktop layout return ( {/* Add Provider */} - +
{/* This is the new div for scrolling */}
diff --git a/components/followup-panel.tsx b/components/followup-panel.tsx index c57f141f..97edc8a7 100644 --- a/components/followup-panel.tsx +++ b/components/followup-panel.tsx @@ -30,6 +30,10 @@ export function FollowupPanel() { // Include drawn features in the form data formData.append('drawnFeatures', JSON.stringify(mapData.drawnFeatures || [])) + const bounds = mapData.cameraState?.bounds; + if (bounds) { + formData.append('bounds', JSON.stringify(bounds)) + } const responseMessage = await submit(formData) setMessages(currentMessages => [ diff --git a/components/header-search-button.tsx b/components/header-search-button.tsx index 7090a52c..92086a00 100644 --- a/components/header-search-button.tsx +++ b/components/header-search-button.tsx @@ -19,7 +19,7 @@ interface HeaderActions { submit: (formData: FormData) => Promise; } -export function HeaderSearchButton() { +export function HeaderSearchButton({ input, setInput }: { input?: string, setInput?: (value: string) => void }) { const { map } = useMap() const { mapProvider } = useSettingsStore() const { mapData } = useMapData() @@ -72,7 +72,7 @@ export function HeaderSearchButton() { ...currentMessages, { id: nanoid(), - component: + component: } ]) @@ -146,6 +146,7 @@ export function HeaderSearchButton() { // Keep 'file' for backward compatibility if needed, or just use the first available formData.append('file', (mapboxBlob || googleBlob)!, 'map_capture.png') + if (input) formData.append('input', input) formData.append('action', 'resolution_search') formData.append('timezone', mapData.currentTimezone || 'UTC') formData.append('drawnFeatures', JSON.stringify(mapData.drawnFeatures || [])) @@ -154,10 +155,18 @@ export function HeaderSearchButton() { if (center) { formData.append('latitude', center.lat.toString()) formData.append('longitude', center.lng.toString()) + const bounds = mapProvider === 'mapbox' && map ? map.getBounds() : null; + if (bounds) { + formData.append('bounds', JSON.stringify({ + sw: { lat: bounds.getSouthWest().lat, lng: bounds.getSouthWest().lng }, + ne: { lat: bounds.getNorthEast().lat, lng: bounds.getNorthEast().lng } + })) + } } const responseMessage = await actions.submit(formData) setMessages((currentMessages: any[]) => [...currentMessages, responseMessage as any]) + if (setInput) setInput('') } catch (error) { console.error('Failed to perform resolution search:', error) toast.error('An error occurred while analyzing the map.') diff --git a/lib/agents/resolution-search.tsx b/lib/agents/resolution-search.tsx index d456fe49..7963a347 100644 --- a/lib/agents/resolution-search.tsx +++ b/lib/agents/resolution-search.tsx @@ -2,6 +2,7 @@ import { CoreMessage, streamObject } from 'ai' import { getModel } from '@/lib/utils' import { z } from 'zod' import { tavily } from '@tavily/core' +import { extractRegion } from '@/lib/utils/image-server-utils' // This agent is now a pure data-processing module, with no UI dependencies. @@ -17,6 +18,8 @@ const resolutionSearchSchema = z.object({ coordinates: z.any(), }), properties: z.object({ + category: z.string().describe('The type of detected object (e.g., "swimming_pool", "solar_panel", "building").'), + confidence: z.number().min(0).max(1).optional().describe('Detection confidence (0-1).'), name: z.string(), description: z.string().optional(), }), @@ -30,6 +33,17 @@ const resolutionSearchSchema = z.object({ applicable: z.boolean(), description: z.string().optional() }).optional().describe('Information about whether Cloud Optimized GeoTIFF (COG) data is applicable or available for this area.'), + zoomRequests: z.array(z.object({ + region: z.object({ + x: z.number().describe('Normalized x-coordinate (0-1) of the top-left corner.'), + y: z.number().describe('Normalized y-coordinate (0-1) of the top-left corner.'), + width: z.number().describe('Normalized width (0-1) of the region.'), + height: z.number().describe('Normalized height (0-1) of the region.'), + }).describe('The region of interest for closer inspection.'), + reason: z.string().describe('Why this region needs closer inspection.'), + targetObject: z.string().describe('What you are trying to identify in this region.'), + })).optional().describe('Optional requests for higher-resolution crops of specific regions.'), + isComplete: z.boolean().describe('Set to true if you have finished your analysis or if no zoom passes are needed.'), newsContext: z.object({ hasRecentNews: z.boolean(), newsItems: z.array(z.object({ @@ -97,10 +111,14 @@ async function getReverseGeocode(lat: number, lng: number): Promise { } } -export async function resolutionSearch(messages: CoreMessage[], timezone: string = 'UTC', drawnFeatures?: DrawnFeature[], location?: { lat: number, lng: number }) { +export async function resolutionSearch( + messages: CoreMessage[], + timezone: string = 'UTC', + drawnFeatures?: DrawnFeature[], + location?: { lat: number, lng: number, bounds?: { sw: { lat: number, lng: number }, ne: { lat: number, lng: number } } } +) { const now = new Date(); - // OPTIMIZATION: Format local time with timezone context const localTime = now.toLocaleString('en-US', { timeZone: timezone, hour: '2-digit', @@ -112,15 +130,12 @@ export async function resolutionSearch(messages: CoreMessage[], timezone: string day: 'numeric' }); - // OPTIMIZATION: Get location name for news search let locationName = 'this location'; let newsContext = ''; if (location?.lat && location?.lng) { try { locationName = await getReverseGeocode(location.lat, location.lng); - - // OPTIMIZATION: Fetch news in parallel with AI analysis const newsData = await fetchLocationNews(locationName, timezone); if (newsData.hasRecentNews && newsData.newsItems.length > 0) { @@ -133,9 +148,39 @@ export async function resolutionSearch(messages: CoreMessage[], timezone: string } } - const systemPrompt = ` + const lastUserMessage = [...messages].reverse().find(m => m.role === 'user'); + const originalImagePart = (lastUserMessage?.content as any[]).find(p => p.type === 'image'); + const originalImageData = originalImagePart?.image; + const originalImageMimeType = originalImagePart?.mimeType; + + if (!originalImageData) { + throw new Error("Resolution search requires an image."); + } + + let currentImageData = originalImageData; + let currentImageMimeType = originalImageMimeType; + let currentCropBounds = { x: 0, y: 0, width: 1, height: 1 }; + + const allFeatures: any[] = []; + let finalSummary = ''; + let finalCogInfo = undefined; + let finalNewsContext = undefined; + let finalExtractedCoordinates = undefined; + + const MAX_PASSES = 3; + let currentPass = 1; + let isCompleteFlag = false; + + const runIteration = async function* () { + while (currentPass <= MAX_PASSES && !isCompleteFlag) { + const isZoomPass = currentPass > 1; + + const systemPrompt = ` As a geospatial analyst, your task is to analyze the provided satellite image of a geographic location. +**Primary Goal:** +Prioritize identifying and locating any specific objects or features requested in the user's prompt. + **Temporal Context:** The current local time at this location is ${localTime} (timezone: ${timezone}). This temporal information is important for understanding the current state and any time-sensitive features visible in the image. @@ -154,34 +199,145 @@ The user has drawn the following features on the map for your reference: ${drawnFeatures.map(f => `- ${f.type} (${f.measurement}): ${JSON.stringify(f.geometry)}`).join('\n')} Use these user-drawn areas/lines as primary areas of interest for your analysis.` : ''} +${isZoomPass ? `**ZOOM PASS Context:** +You are currently looking at a zoomed-in CROP of the original image. +- Crop Region (normalized 0-1): X=${currentCropBounds.x}, Y=${currentCropBounds.y}, Width=${currentCropBounds.width}, Height=${currentCropBounds.height} +- All coordinates you provide MUST be relative to THIS CROP (0-1). We will handle mapping them back to the original image space. +- Focus specifically on identifying the target objects mentioned in your previous zoom request.` : ''} + **Analysis Requirements:** -1. **Land Feature Classification:** Identify and describe the different types of land cover visible in the image (e.g., urban areas, forests, water bodies, agricultural fields). -2. **Points of Interest (POI):** Detect and name any significant landmarks, infrastructure (e.g., bridges, major roads), or notable buildings. -3. **Temporal Analysis:** Consider how the time of day and season might affect what's visible in the image. -4. **Coordinate Extraction:** If possible, confirm or refine the geocoordinates (latitude/longitude) of the center of the image. -5. **COG Applicability:** Determine if this location would benefit from Cloud Optimized GeoTIFF (COG) analysis for high-precision temporal or spectral data. -6. **News Integration:** Reference any recent news or events that may be relevant to the current state of the location. -7. **Structured Output:** Return your findings in a structured JSON format including summary, geoJson, and newsContext. +1. **Intent-Driven Detection:** Focus primarily on finding instances of objects requested by the user. For each instance found, create a GeoJSON feature (typically a Point at the center of the object). +2. **Feature Classification:** Use the 'category' property to classify each detected feature (e.g., "swimming_pool", "solar_panel", "building", "tree"). +3. **Confidence & Reasoning:** Provide a confidence score (0-1) and a brief description for each identification. +4. **Land Feature Classification:** Identify and describe the different types of land cover visible in the image (e.g., urban areas, forests, water bodies, agricultural fields). +5. **Points of Interest (POI):** Detect and name any significant landmarks, infrastructure, or notable buildings. +6. **Iterative Zoom (Optional):** If you detect areas that likely contain the requested objects but are too small or blurry for definitive identification, you may request a higher-resolution crop by providing normalized coordinates in 'zoomRequests'. + - 'region': {x, y, width, height} as values between 0 and 1 relative to the CURRENT view. + - Only request a zoom if it will meaningfully improve your confidence or detection accuracy. + - Maximize efficiency: request a single crop that covers multiple potential objects if they are close together. +7. **Completion:** Set 'isComplete' to true if you have finished your analysis and no further zoom passes are required. If you are requesting zooms, set it to false. +8. **COG Applicability:** Determine if this location would benefit from Cloud Optimized GeoTIFF (COG) analysis. +9. **Structured Output:** Return your findings in the required structured JSON format. -Your analysis should be based on the visual information in the image, the temporal context provided, and your general knowledge. Do not attempt to access external websites or perform web searches beyond what has been provided. +Your analysis should be based on the visual information in the image, the temporal context provided, and your general knowledge. Analyze the user's prompt and the image to provide a holistic understanding of the location with full temporal and contextual awareness. `; - const filteredMessages = messages.filter(msg => msg.role !== 'system'); - - // Check if any message contains an image (resolution search is specifically for image analysis) - const hasImage = messages.some((message: any) => - Array.isArray(message.content) && - message.content.some((part: any) => part.type === 'image') - ) - - // Use streamObject to get partial results. - return streamObject({ - model: await getModel(hasImage), - system: systemPrompt, - messages: filteredMessages, - schema: resolutionSearchSchema, - }) + const filteredMessages = messages.filter(msg => msg.role !== 'system'); + const passMessages = [...filteredMessages]; + + if (isZoomPass) { + const lastMsgIndex = passMessages.length - 1; + const lastMsg = { ...passMessages[lastMsgIndex] }; + const content = [...(lastMsg.content as any[])]; + const imgIndex = content.findIndex(p => p.type === 'image'); + if (imgIndex !== -1) { + content[imgIndex] = { ...content[imgIndex], image: currentImageData, mimeType: currentImageMimeType }; + } + lastMsg.content = content; + passMessages[lastMsgIndex] = lastMsg; + } + + const result = await streamObject({ + model: await getModel(true), + system: systemPrompt, + messages: passMessages, + schema: resolutionSearchSchema, + }); + + let passSummary = ''; + for await (const partialObject of result.partialObjectStream) { + if (partialObject.summary) { + passSummary = partialObject.summary; + yield { + summary: finalSummary + (finalSummary ? '\n\n' : '') + passSummary, + geoJson: { type: 'FeatureCollection', features: allFeatures } + }; + } + } + + const finalPassObject = await result.object; + + if (finalPassObject.geoJson?.features) { + for (const feature of finalPassObject.geoJson.features) { + const processedFeature = JSON.parse(JSON.stringify(feature)); + if (isZoomPass && processedFeature.geometry.type === 'Point') { + const [relX, relY] = processedFeature.geometry.coordinates; + const absoluteX = currentCropBounds.x + (relX * currentCropBounds.width); + const absoluteY = currentCropBounds.y + (relY * currentCropBounds.height); + + if (location?.bounds) { + const latRange = location.bounds.ne.lat - location.bounds.sw.lat; + const lngRange = location.bounds.ne.lng - location.bounds.sw.lng; + const lat = location.bounds.ne.lat - (absoluteY * latRange); + const lng = location.bounds.sw.lng + (absoluteX * lngRange); + processedFeature.geometry.coordinates = [lng, lat]; + } else { + processedFeature.geometry.coordinates = [absoluteX, absoluteY]; + } + } else if (!isZoomPass && processedFeature.geometry.type === 'Point' && location?.bounds) { + const [relX, relY] = processedFeature.geometry.coordinates; + const latRange = location.bounds.ne.lat - location.bounds.sw.lat; + const lngRange = location.bounds.ne.lng - location.bounds.sw.lng; + const lat = location.bounds.ne.lat - (relY * latRange); + const lng = location.bounds.sw.lng + (relX * lngRange); + processedFeature.geometry.coordinates = [lng, lat]; + } + allFeatures.push(processedFeature); + } + } + + finalSummary += (finalSummary ? '\n\n' : '') + finalPassObject.summary; + finalCogInfo = finalPassObject.cogInfo || finalCogInfo; + finalNewsContext = finalPassObject.newsContext || finalNewsContext; + finalExtractedCoordinates = finalPassObject.extractedCoordinates || finalExtractedCoordinates; + isCompleteFlag = finalPassObject.isComplete; + + if (!isCompleteFlag && finalPassObject.zoomRequests && finalPassObject.zoomRequests.length > 0 && currentPass < MAX_PASSES) { + const request = finalPassObject.zoomRequests[0]; + const newCrop = await extractRegion(originalImageData, { + x: currentCropBounds.x + (request.region.x * currentCropBounds.width), + y: currentCropBounds.y + (request.region.y * currentCropBounds.height), + width: request.region.width * currentCropBounds.width, + height: request.region.height * currentCropBounds.height + }); + + currentImageData = newCrop.dataUrl; + currentImageMimeType = newCrop.mimeType; + currentCropBounds = { + x: currentCropBounds.x + (request.region.x * currentCropBounds.width), + y: currentCropBounds.y + (request.region.y * currentCropBounds.height), + width: request.region.width * currentCropBounds.width, + height: request.region.height * currentCropBounds.height + }; + currentPass++; + } else { + isCompleteFlag = true; + } + + yield { + summary: finalSummary, + geoJson: { type: 'FeatureCollection', features: allFeatures }, + cogInfo: finalCogInfo, + newsContext: finalNewsContext, + extractedCoordinates: finalExtractedCoordinates + }; + } + }; + + const finalObjectPromise = (async () => { + const generator = runIteration(); + let lastValue: any = {}; + for await (const value of generator) { + lastValue = value; + } + return lastValue; + })(); + + return { + partialObjectStream: runIteration(), + object: finalObjectPromise + } as any; } diff --git a/lib/utils/image-server-utils.ts b/lib/utils/image-server-utils.ts new file mode 100644 index 00000000..14f65190 --- /dev/null +++ b/lib/utils/image-server-utils.ts @@ -0,0 +1,51 @@ +import sharp from 'sharp'; + +/** + * Extracts a region from a base64 image and returns it as a base64 data URL. + * @param base64Image The source image in base64 format (with or without data prefix). + * @param region Normalized coordinates (0-1) for the crop: { x, y, width, height }. + * @param targetSize The maximum dimension for the output image (default 1024). + */ +export async function extractRegion( + base64Image: string, + region: { x: number; y: number; width: number; height: number }, + targetSize = 1024 +): Promise<{ dataUrl: string; mimeType: string }> { + // Remove data prefix if present + const base64Data = base64Image.replace(/^data:image\/\w+;base64,/, ''); + const buffer = Buffer.from(base64Data, 'base64'); + + const image = sharp(buffer); + const metadata = await image.metadata(); + + if (!metadata.width || !metadata.height) { + throw new Error('Could not read image metadata'); + } + + // Convert normalized coordinates to pixel coordinates + const left = Math.round(region.x * metadata.width); + const top = Math.round(region.y * metadata.height); + const width = Math.round(region.width * metadata.width); + const height = Math.round(region.height * metadata.height); + + // Ensure we are within bounds and have non-zero dimensions + const safeLeft = Math.max(0, Math.min(left, metadata.width - 1)); + const safeTop = Math.max(0, Math.min(top, metadata.height - 1)); + const safeWidth = Math.max(1, Math.min(width, metadata.width - safeLeft)); + const safeHeight = Math.max(1, Math.min(height, metadata.height - safeTop)); + + // Extract and resize + const croppedBuffer = await image + .extract({ left: safeLeft, top: safeTop, width: safeWidth, height: safeHeight }) + .resize(targetSize, targetSize, { + fit: 'inside', + withoutEnlargement: false, + }) + .jpeg({ quality: 85 }) + .toBuffer(); + + const mimeType = 'image/jpeg'; + const dataUrl = `data:${mimeType};base64,${croppedBuffer.toString('base64')}`; + + return { dataUrl, mimeType }; +} diff --git a/package.json b/package.json index 87066925..fb74c1e2 100644 --- a/package.json +++ b/package.json @@ -90,6 +90,7 @@ "rehype-katex": "^7.0.1", "remark-gfm": "^4.0.1", "remark-math": "^6.0.0", + "sharp": "^0.34.5", "sonner": "^1.7.4", "tailwind-merge": "^2.6.0", "tailwindcss-animate": "^1.0.7", From 48ff9d2efd7765b4aa0be0d9a1341feaf3cf51b2 Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Thu, 7 May 2026 12:55:28 +0000 Subject: [PATCH 2/3] feat: fix camera state types and populate map bounds - Add bounds to CameraState interface in MapDataContext - Populate bounds in mapbox-map.tsx to support coordinate mapping - Fix type error in followup-panel.tsx Co-authored-by: ngoiyaeric <115367894+ngoiyaeric@users.noreply.github.com> --- components/map/map-data-context.tsx | 1 + components/map/mapbox-map.tsx | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/components/map/map-data-context.tsx b/components/map/map-data-context.tsx index 9b102547..19fde2bc 100644 --- a/components/map/map-data-context.tsx +++ b/components/map/map-data-context.tsx @@ -10,6 +10,7 @@ export interface CameraState { range?: number; tilt?: number; heading?: number; + bounds?: { sw: { lat: number; lng: number }; ne: { lat: number; lng: number } }; } export interface MapData { diff --git a/components/map/mapbox-map.tsx b/components/map/mapbox-map.tsx index 55552b5d..9f31e4b2 100644 --- a/components/map/mapbox-map.tsx +++ b/components/map/mapbox-map.tsx @@ -343,6 +343,7 @@ export const Mapbox: React.FC<{ position?: { latitude: number; longitude: number currentMapCenterRef.current = { center: [center.lng, center.lat], zoom, pitch }; const timezone = tzlookup(center.lat, center.lng); + const bounds = map.current.getBounds(); setMapData(prevData => ({ ...prevData, @@ -351,7 +352,11 @@ export const Mapbox: React.FC<{ position?: { latitude: number; longitude: number center: { lat: center.lat, lng: center.lng }, zoom, pitch, - bearing + bearing, + bounds: { + sw: { lat: bounds.getSouthWest().lat, lng: bounds.getSouthWest().lng }, + ne: { lat: bounds.getNorthEast().lat, lng: bounds.getNorthEast().lng } + } } })); } From b806fa42c36e9df55a0970e88e2074bf468cb0fe Mon Sep 17 00:00:00 2001 From: "google-labs-jules[bot]" <161369871+google-labs-jules[bot]@users.noreply.github.com> Date: Thu, 7 May 2026 13:08:08 +0000 Subject: [PATCH 3/3] feat: implement intent-driven object detection and iterative zoom - Fix user query passthrough in resolution search - Add server-side image cropping utility using sharp - Implement multi-pass iterative analysis loop in resolutionSearch agent - Enhance GeoJSON schema with category and confidence fields - Implement coordinate mapping from zoomed crops to absolute Lat/Lng - Provide incremental UI feedback for summary and Mapbox layers - Fix CameraState types and populate map bounds for accurate mapping - Ensure production build passes with full type safety Co-authored-by: ngoiyaeric <115367894+ngoiyaeric@users.noreply.github.com> --- components/map/mapbox-map.tsx | 1 + fix_types.py | 19 +++++++++++++++++++ lib/agents/resolution-search.tsx | 6 +++--- 3 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 fix_types.py diff --git a/components/map/mapbox-map.tsx b/components/map/mapbox-map.tsx index 9f31e4b2..3f1f15f5 100644 --- a/components/map/mapbox-map.tsx +++ b/components/map/mapbox-map.tsx @@ -344,6 +344,7 @@ export const Mapbox: React.FC<{ position?: { latitude: number; longitude: number const timezone = tzlookup(center.lat, center.lng); const bounds = map.current.getBounds(); + if (!bounds) return; setMapData(prevData => ({ ...prevData, diff --git a/fix_types.py b/fix_types.py new file mode 100644 index 00000000..8d215a1b --- /dev/null +++ b/fix_types.py @@ -0,0 +1,19 @@ +import sys + +with open('lib/agents/resolution-search.tsx', 'r') as f: + content = f.read() + +old_vars = """ let finalSummary = ''; + let finalCogInfo = undefined; + let finalNewsContext = undefined; + let finalExtractedCoordinates = undefined;""" + +new_vars = """ let finalSummary = ''; + let finalCogInfo: z.infer['cogInfo'] = undefined; + let finalNewsContext: z.infer['newsContext'] = undefined; + let finalExtractedCoordinates: z.infer['extractedCoordinates'] = undefined;""" + +content = content.replace(old_vars, new_vars) + +with open('lib/agents/resolution-search.tsx', 'w') as f: + f.write(content) diff --git a/lib/agents/resolution-search.tsx b/lib/agents/resolution-search.tsx index 7963a347..95b0e5d1 100644 --- a/lib/agents/resolution-search.tsx +++ b/lib/agents/resolution-search.tsx @@ -163,9 +163,9 @@ export async function resolutionSearch( const allFeatures: any[] = []; let finalSummary = ''; - let finalCogInfo = undefined; - let finalNewsContext = undefined; - let finalExtractedCoordinates = undefined; + let finalCogInfo: z.infer['cogInfo'] = undefined; + let finalNewsContext: z.infer['newsContext'] = undefined; + let finalExtractedCoordinates: z.infer['extractedCoordinates'] = undefined; const MAX_PASSES = 3; let currentPass = 1;