From 84e5788d36b2e80012442e78a9863b63c457a2c0 Mon Sep 17 00:00:00 2001 From: Yuan SY Date: Thu, 9 Jul 2026 13:49:44 +0800 Subject: [PATCH] Refine cart follow gating and diagnostics --- android/cartfollow-devlog.md | 140 +++++++- .../point_goal_navigation_source_analysis.md | 307 ++++++++++++++++++ .../openbot/cartfollow/ActionArbitrator.java | 10 +- .../cartfollow/BboxContinuityEvidence.java | 10 + .../cartfollow/FollowStateMachine.java | 26 +- .../HumanCartSimulatorFragment.java | 112 ++++++- .../cartfollow/IdentityBeliefAccumulator.java | 121 ++++++- .../openbot/cartfollow/IdentityEvidence.java | 4 + .../cartfollow/TargetTrackManager.java | 154 ++++++++- .../CartFollowDiagnosticSaver.java | 5 +- .../CartFollowDiagnosticSession.java | 2 +- .../layout/fragment_human_cart_simulator.xml | 9 + 12 files changed, 868 insertions(+), 32 deletions(-) create mode 100644 android/point_goal_navigation_source_analysis.md diff --git a/android/cartfollow-devlog.md b/android/cartfollow-devlog.md index d842e20ff..03578a026 100644 --- a/android/cartfollow-devlog.md +++ b/android/cartfollow-devlog.md @@ -3,7 +3,7 @@ > 所属项目:自主跟随购物车原型 > 代码位置:`dev/OpenBot/android/robot/src/main/java/org/openbot/cartfollow/` > 开发分支:`feature/human-cart-simulator`(Phase 1)/ `feature/distance-control`(Phase 2)/ `feature/person-crop-collector`(Phase 3 起) -> 最后更新:2026-07-08 +> 最后更新:2026-07-09 --- @@ -87,7 +87,8 @@ Human Cart Simulator 是购物车跟随功能的上位机核心模块,在 Open | **Person Sequence Collector** | 已完成 | 可采集无人帧、多人检测、bbox、crop 和人工事件,用于 PC sequence replay | | **阶段 A 行为层** | 已完成并通过手机体验 | `Evidence -> BehaviorDecisionResult -> BehaviorAction -> HumanCommand` 已接入 Human Cart Simulator | | **阶段 B Android ReID** | 已完成首版并通过手机运行 | TFLite ReID 可运行,debug 字段正常,实机约 30 FPS;仍需阶段 C 轨迹与身份信念层抑制跟错人 | -| **阶段 C 目标轨迹与身份信念层** | 已完成首版代码接入 | 新增短时 trackId、lockedTrackId、targetBelief 和 suspectedTrack debug,待手机验收调参 | +| **阶段 C 目标轨迹与身份信念层** | 已完成策略修正版代码接入 | 已包含短时 trackId、lockedTrackId、targetBelief、suspectedTrack、locked ghost、suspected 滞回、loose/default/strict bbox gate、恢复后 relock 与非 locked 空间支持门控,待安装新版 APK 后复测 | +| **诊断日志开关** | 已完成 | Human Cart Simulator 新增“记录日志”开关,默认关闭;关闭时不创建 diagnostics session,不写 CSV/JSON/crop/gallery/event | ### 2.2 Phase 3 首轮 ReID 实验结果(2026-07-06) @@ -883,3 +884,138 @@ gallery_candidate_landscape_rate 是否保持 0 ``` 如果 `candidate_switch_penalty` 和 `belief_high_bbox_failed` 下降,同时非目标转绿不增加,才说明策略改动真正改善了“目标返回后迟迟不转绿”和“干扰者偶发转绿”两个问题。 + +--- + +## 15. Phase C 下一轮策略版本:track/bbox gate 小步修正(2026-07-08) + +### 15.1 开发定位 + +阶段 C 首版已经接入 `TargetTrackManager + IdentityBeliefAccumulator`,下一轮不再新增大架构,而是优化 track 粘性、locked/suspected 保护和 bbox gate 的状态化使用。 + +状态更新:本节为 2026-07-08 的实施计划,2026-07-09 已完成代码接入;最新实现状态见第 16 节。 + +本轮不做: + +```text +更换 ReID 模型 +启用 dynamic gallery +全局放宽 bbox gate +接通真实底盘前进 +优先做复杂避障 +``` + +### 15.2 计划代码点(已在第 16 节落地) + +| 文件 | 改动 | +|------|------| +| `TargetTrackManager.java` | 增加 locked track ghost memory、suspected track 最小滞回、疑似目标替换门槛。 | +| `BboxContinuityEvidence.java` | 在 default / strict 之外新增 loose admission gate。 | +| `IdentityBeliefAccumulator.java` | 分离 identity belief 与 motion permission;belief 高但 bbox failed 时保留 suspected,不直接清零。 | +| `FollowStateMachine.java` | 在 `IDENTITY_UNCERTAIN / SEARCH / REACQUIRE_TARGET` 使用 admission/confirm/motion 分层恢复。 | +| diagnostics/debug 字段 | 输出 `loose_admission_only / motion_gate_failed / suspected_dwell_hold / locked_ghost_reference` 等 reason。 | + +### 15.3 策略口径 + +恢复链路固定为: + +```text +loose admission + -> suspectedTrack + -> default confirm + -> REACQUIRE_TARGET + -> strict/default motion gate + -> FOLLOW_CAUTION / FOLLOW +``` + +具体原则: + +- loose gate 只允许进入 suspected/reacquire,不允许直接触发前进。 +- `FOLLOW / FOLLOW_CAUTION` 继续使用 default/strict bbox gate、prediction 或 locked track 保护,保持运动门槛保守。 +- `IDENTITY_UNCERTAIN / SEARCH` 可以接纳 loose admission 的疑似目标,但 action 仍为 `MOTION_STOP / LOCAL_SEARCH`。 +- `belief 高 + bbox failed` 表示身份可能正确但运动暂不安全,应保留 belief 或轻微衰减,并保持 `REACQUIRE_HOLD / MOTION_STOP`。 +- hard `STOP` 仍只由搜索超时、急停、通信异常、高风险障碍或人工取消触发。 + +### 15.4 复测场景 + +下一轮 APK 构建后采集 4 段短日志,每段 30-60 秒: + +1. 目标离开后原目标返回。 +2. 目标离开后干扰者进入。 +3. 目标在场时干扰者穿越。 +4. 目标短遮挡、蹲下或局部可见。 + +PC compare 验收指标: + +```text +recovered_rate 不下降; +mean_ms_to_follow 下降或不恶化; +not_recovered_in_window 减少; +candidate_switch_penalty 下降; +belief_high_bbox_failed 下降; +非目标转绿不增加; +hard_stop_count 不增加; +gallery_candidate_landscape_rate 保持 0。 +``` + +--- + +## 16. Phase C relock、空间支持门控与日志开关实现(2026-07-09) + +### 16.1 本轮实现状态 + +本轮 Android 代码已完成接入,目标是修正 2026-07-09 新采集数据暴露的两个问题:目标返回后新 track 没有晋升为 locked track,以及干扰者在没有 bbox / ghost 空间支持时仍可能靠 ReID 抬高 belief。 + +本轮仍保持边界不变: + +```text +不更换 ReID 模型 +不启用 dynamic gallery +不开放真实底盘前进 +不改 ControlGenerator 的真实底盘控制路径 +不提前引入复杂避障 +``` + +### 16.2 恢复后 relock + +- 非 locked track 在 `REACQUIRE_TARGET / READY_TO_FOLLOW / FOLLOW_CAUTION / FOLLOW` 的安全恢复路径中,如果连续通过 motion gate,并在 `FOLLOW_CAUTION / FOLLOW_SLOW` 等保守动作下稳定至少 2 帧,可以晋升为新的 `lockedTrackId`。 +- relock 成功后同步调用 `IdentityBeliefAccumulator.lockTrack(newTrackId)`,清空旧 suspected track。 +- debug / diagnostic reason 输出 `relock_after_recovery`,用于 PC 侧确认目标返回后是否真正完成重新锁定。 +- 普通 `IDENTITY_UNCERTAIN` 中单帧高 ReID 分数不允许直接 relock。 + +### 16.3 非 locked 空间支持门控 + +- 非 locked track 只有满足 `bboxLoose / bboxDefault / prediction / nearLockedGhost` 之一,才允许成为 suspected target。 +- 如果只有 ReID 高分但空间支持全 false,记录 `reid_interest_no_spatial_support` 或 `spatial_support_missing`。 +- 无空间支持时允许保留低强度观察信号,但 belief 不得升到 caution / confirm 阈值,也不能触发 suspected 或 FOLLOW。 +- `candidateSwitchCount` 只在真正选中或切换 suspected / selected track 时增长,减少“观察到高分干扰者”导致的噪声。 + +### 16.4 诊断日志开关 + +- `fragment_human_cart_simulator.xml` 底部控制区新增 `diagnostic_switch`,文案为“记录日志”,默认关闭。 +- `HumanCartSimulatorFragment` 新增 `diagnosticEnabled` 状态。 +- 日志关闭时,`startDiagnosticSession()` 和 `activateDiagnosticSession()` 直接返回,不创建 `cartfollow_diagnostics` session,不写 `session_info.json`,不保存 confirmed snapshot、gallery、crop、CSV、events 或 session_stop。 +- 运行中关闭日志会立即停止 active session,并禁用“目标离开 / 返回”事件按钮。 +- 日志开启并 Confirm 激活 session 后,事件按钮才可用;跟随推理、状态机和 debug 文本不受日志开关影响。 + +### 16.5 构建验证 + +构建命令: + +```powershell +$env:JAVA_HOME='C:\Users\ysyys\.jdks\jbr-17.0.14' +$env:Path="$env:JAVA_HOME\bin;$env:Path" +.\gradlew.bat :robot:assembleDebug +``` + +结果:构建通过,生成 debug APK。构建日志仍包含既有 TensorFlow Lite namespace、Kotlin/Javac target 和 deprecated API warning,未阻塞构建。 + +### 16.6 下一轮手机验收重点 + +1. 日志开关关闭时,Start、Confirm、运行、Retake、Cancel 都不应新增 `cartfollow_diagnostics/cart_diag_*` 目录。 +2. 日志开关开启后,Confirm 才激活 session,并正常生成 `frame_log.csv / identity_log.csv / events.csv / session_info.json / gallery / crops`。 +3. 目标离开后原目标返回:期望恢复到新 track 后自动 relock,后续 `trackId == lockedTrackId`。 +4. 蹲下、弯腰或局部遮挡:期望检测器丢框后可恢复,恢复成功后 relock 到新 track。 +5. 目标离开后干扰者进入:期望干扰者无空间支持时不进入 suspected,不恢复 FOLLOW。 +6. 目标在场时干扰者穿越:期望 `candidateSwitchCount` 明显下降,非 locked FOLLOW 行数减少或归零。 +7. PC compare 继续检查 `candidate_switch_penalty / belief_high_bbox_failed / recovered_rate / hard_stop_count / gallery_candidate_landscape_rate`,合格后再讨论极低速真实底盘联调。 diff --git a/android/point_goal_navigation_source_analysis.md b/android/point_goal_navigation_source_analysis.md new file mode 100644 index 000000000..eb5b71c42 --- /dev/null +++ b/android/point_goal_navigation_source_analysis.md @@ -0,0 +1,307 @@ +# OpenBot PointGoalNavigation 源码解析 + +本文基于当前 `dev/OpenBot/android` 源码,解释 OpenBot Android 端 `Point Goal Navigation` 功能的实际实现方式。这个功能不能简单理解成传统地图导航或 SLAM 导航;它更像是“ARCore 相对位姿 + 相机图像 + 学习策略网络”的点目标视觉导航 demo。 + +## 一句话结论 + +`PointGoalNavigation` 让用户输入一个相对于机器人当前位姿的二维目标点,例如“前方 1.0 m、左侧 0 m”。启动后,Android 手机用 ARCore 持续估计当前手机/机器人位姿,再把当前相机图像、目标距离、目标相对朝向送入 `navigation.tflite`。模型直接输出左右轮控制量,OpenBot 再通过 `Vehicle.setControl()` 下发给底盘。 + +也就是说,它不是靠地图、路径规划或障碍物几何建模来导航,而是依赖一个训练好的 TFLite 策略网络端到端地产生左右轮速度。 + +## 关键源码位置 + +- 功能入口:`robot/src/main/java/org/openbot/pointGoalNavigation/PointGoalNavigationFragment.java` +- ARCore 封装:`robot/src/main/java/org/openbot/pointGoalNavigation/ArCore.java` +- TFLite 策略封装:`robot/src/main/java/org/openbot/tflite/Navigation.java` +- 目标输入弹窗:`robot/src/main/java/org/openbot/pointGoalNavigation/SetGoalDialogFragment.java` +- UI 布局:`robot/src/main/res/layout/fragment_point_goal_navigation.xml` +- 目标输入布局:`robot/src/main/res/layout/set_goal_dialog_view.xml` +- 功能入口菜单:`robot/src/main/java/org/openbot/common/FeatureList.java` +- Fragment 路由:`robot/src/main/res/navigation/nav_graph.xml` +- 模型配置:`robot/src/main/assets/config.json` +- 内置模型:`robot/src/main/assets/networks/navigation.tflite` + +## 功能入口与运行前提 + +主界面功能列表中有一个 `Point Goal Navigation` 子项。点击后,`MainFragment` 会跳转到 `PointGoalNavigationFragment`。 + +这个 Fragment 的界面本身非常简单,只有一个全屏 `GLSurfaceView`,用于显示 ARCore 相机背景和目标标记。真正的逻辑不在 XML,而在 `PointGoalNavigationFragment` 和 `ArCore` 两个 Java 类里。 + +运行前提有三类: + +1. Android 相机权限。 +2. 设备支持 ARCore,并且系统中可用 Google Play Services for AR。 +3. OpenBot 车辆通信链路可用,因为最终控制会调用 `Vehicle.setControl()`。 + +项目的 `robot/build.gradle` 依赖了 `com.google.ar:core:1.29.0`,`AndroidManifest.xml` 中也声明了 `com.google.ar.core` 为 `optional`。但是代码里创建 ARCore `Session` 时仍然需要运行时 ARCore 服务。如果手机没有可用的 ARCore 运行环境,`setupArCore()` 会捕获 `UnavailableArcoreNotInstalledException`、`UnavailableDeviceNotCompatibleException`、`UnavailableApkTooOldException` 等异常,并弹出 ARCore failure 信息。 + +这解释了为什么在中国大陆没有 Google Play 服务时很难直接实机测试:模型文件本身是内置的,不需要联网下载;真正卡住的是 ARCore 运行服务和设备兼容链路。 + +## 用户输入的目标是什么 + +进入功能后会弹出 `Set Goal` 对话框。用户填写两个数: + +- `Forward [m]`:目标在机器人前方多少米,默认 `1.0`。 +- `Left [m]`:目标在机器人左侧多少米,默认 `0`。 + +这两个值允许为负数。README 中也说明:负的 forward 表示后方,负的 left 表示右侧。 + +源码中点击 `Start` 后会调用: + +```java +startDriving(-left, -forward); +``` + +随后在 `startDriving()` 中有注释: + +```java +// x: right, z: backwards +``` + +也就是说,用户用“前/左”的直觉坐标输入,代码再转换成 ARCore/OpenBot 使用的坐标: + +- ARCore/OpenBot 这里把 `x` 轴正方向当作右侧。 +- 把 `z` 轴正方向当作后方。 +- 因此前方目标要变成负 `z`,左侧目标要变成负 `x`。 + +例如默认输入 `Forward = 1.0, Left = 0`,实际目标平移就是 `(x=0, z=-1.0)`,即当前位姿前方 1 米。 + +## ARCore 在这里做了什么 + +`ArCore` 类负责封装 ARCore `Session`、相机画面、位姿和目标锚点。它不做 SLAM 地图输出,也不做路径规划;它主要提供两件事: + +1. 当前手机/机器人位姿 `currentPose`。 +2. 用户目标点的 ARCore 锚点 `targetAnchor`。 + +启动驾驶时,代码先: + +1. 清除旧锚点。 +2. 把当前 ARCore pose 保存为起点锚点。 +3. 用起点 pose 叠加用户输入的平移,创建目标锚点。 + +核心逻辑是: + +```java +arCore.setStartAnchorAtCurrentPose(); +Pose startPose = arCore.getStartPose(); +arCore.setTargetAnchor(startPose.compose(Pose.makeTranslation(goalX, 0.0f, goalZ))); +``` + +之后每一帧,`ArCore.onDrawFrame()` 会: + +1. 调用 `session.update()` 获取 ARCore 帧。 +2. 检查相机 tracking 状态。 +3. 从 ARCore camera 读取当前 pose。 +4. 读取目标 anchor pose。 +5. 调用 `frame.acquireCameraImage()` 获取 CPU 图像。 +6. 把 `NavigationPoses`、`ImageFrame`、相机内参和时间戳回调给 `PointGoalNavigationFragment.onArCoreUpdate()`。 + +如果 ARCore 不再处于 `TRACKING` 状态,或者相机会话暂停、不可用,Fragment 会停止底盘并弹窗提示。 + +## 相机配置和图像预处理 + +`ArCore.setCameraConfig()` 明确选择: + +- 后置摄像头。 +- 30 FPS。 +- CPU 图像尺寸必须是 `640x480`。 +- 如果有多个可用配置,选择 GPU texture size 更小的配置。 + +每次导航更新时,`PointGoalNavigationFragment` 会把 ARCore 的 `YUV_420_888` 图像转成 RGB bitmap: + +1. 原始 CPU 图像尺寸通常为 `640x480`。 +2. 按 `160 / 480 = 1/3` 缩放,得到约 `213x160`。 +3. 再裁剪 `Bitmap.createBitmap(bitmap, 0, 30, 160, 90)`,得到模型需要的 `160x90` 图像。 + +所以模型输入图像不是完整相机画面,而是缩放后从左上 x=0、y=30 开始截取的 `160x90` 区域。这个裁剪方式是硬编码的,和 `navigation.tflite` 的训练输入尺寸匹配。 + +像素进入 TFLite 前会在 `Navigation.addPixelValue()` 中归一化到 `[0, 1]`: + +```java +channel = channel / 255.0f +``` + +## 模型输入与输出 + +`config.json` 中配置了一个导航模型: + +```json +{ + "class": "NAVIGATION", + "type": "GOALNAV", + "name": "PilotNet-Goal.tflite", + "path": "networks/navigation.tflite", + "inputSize": "160x90" +} +``` + +不过 `PointGoalNavigationFragment.startDriving()` 并没有从模型管理器动态选择模型,而是直接构造了一个固定模型对象: + +```java +new Model( + 0, + CLASS.NAVIGATION, + TYPE.GOALNAV, + "navigation.tflite", + PATH_TYPE.ASSET, + "networks/navigation.tflite", + "160x90"); +``` + +`Navigation.java` 中可以看到模型有两个输入: + +- `serving_default_img_input:0`:形状必须是 `[1, 90, 160, 3]`。 +- `serving_default_goal_input:0`:3 个 float,分别是: + - `goalDistance` + - `sin(deltaYaw)` + - `cos(deltaYaw)` + +其中 `goalDistance` 是当前 pose 到目标 pose 在水平面上的欧氏距离,只使用 `x` 和 `z`: + +```java +sqrt((goal.x - current.x)^2 + (goal.z - current.z)^2) +``` + +`deltaYaw` 是机器人当前朝向和“指向目标方向”之间的偏航角差。代码不是直接输入角度,而是输入 `sin(deltaYaw)` 和 `cos(deltaYaw)`,这样可以避免角度在 `pi` 和 `-pi` 边界附近的不连续问题。 + +模型输出是一个 `float[1][2]`: + +```java +float[][] predicted_ctrl = new float[1][2]; +``` + +这两个值会被包装成: + +```java +new Control(predicted_ctrl[0][0], predicted_ctrl[0][1]) +``` + +也就是左轮和右轮的归一化控制量。`Control` 构造函数会把每个值夹到 `[-1, 1]`。`Vehicle.sendControl()` 再乘以 `speedMultiplier`,默认代码中是 `192`,然后通过串口协议发送: + +```text +c,\n +``` + +因此,PointGoalNavigation 的控制输出不是“线速度 + 角速度”,而是直接输出 OpenBot 差速底盘左右轮命令。 + +## 每帧控制循环 + +完整闭环可以概括为: + +```mermaid +flowchart TD + A["用户输入 Forward / Left"] --> B["转换为 ARCore 相对目标坐标"] + B --> C["创建 startAnchor 和 targetAnchor"] + C --> D["ARCore 每帧估计 currentPose"] + D --> E["获取 640x480 YUV 相机帧"] + E --> F["转 RGB、缩放、裁剪为 160x90"] + D --> G["计算 goalDistance 和 deltaYaw"] + F --> H["navigation.tflite"] + G --> H + H --> I["输出 left / right"] + I --> J["Vehicle.setControl"] + J --> K["串口发送 c,"] +``` + +对应到 `PointGoalNavigationFragment.onArCoreUpdate()`,每帧做的事情是: + +1. 如果 `isRunning == false`,直接忽略。 +2. 计算当前到目标距离。 +3. 如果距离小于 `0.15 m`,认为到达目标,停止底盘。 +4. 否则计算目标相对偏航。 +5. 取当前相机图像并转成 `160x90` bitmap。 +6. 调用 `navigationPolicy.recognizeImage(bitmap, distance, sinYaw, cosYaw)`。 +7. 把模型输出 `Control` 交给 `vehicle.setControl(control)`。 + +## 停车与异常处理 + +这个功能的安全逻辑比较简单,主要有三类停止: + +1. 到达目标:当 `goalDistance < 0.15f` 时,调用 `stop()`,播放 `Goal reached.`,并弹出信息框。 +2. ARCore tracking 丢失:`onArCoreTrackingFailure()` 中调用 `stop()`,播放 `Tracking lost.`,并弹出信息框。 +3. ARCore session 暂停:`onArCoreSessionPaused()` 中调用 `stop()`,播放 `AR Core session paused.`,并弹出信息框。 + +`stop()` 的动作是: + +```java +arCore.detachAnchors(); +vehicle.stopBot(); +isRunning = false; +``` + +这里没有复杂状态机,也没有短时重定位、避障优先级、通信异常仲裁等逻辑。避障能力如果存在,主要隐含在 `navigation.tflite` 策略网络的训练行为里,而不是显式代码规则。 + +## 它没有做什么 + +结合源码可以明确几点: + +- 没有构建全局地图。 +- 没有使用 SLAM 路径规划。 +- 没有 A*、DWA、TEB 等传统局部规划器。 +- 没有显式障碍物检测模块。 +- 没有用深度图参与当前控制。`DepthFrame` 类存在,但当前 `ArCore` 配置没有启用 depth,`PointGoalNavigationFragment` 也没有使用 `DepthFrame`。 +- 没有用相机内参参与当前控制。`CameraIntrinsics` 会被回调传入,但 `onArCoreUpdate()` 中没有实际使用它。 +- 没有用 Google Maps 或网络地图。所谓 point goal 是相对当前位置的局部目标点,不是地图上的经纬度点。 + +## 对本项目的意义 + +对“自主跟随购物车”项目来说,PointGoalNavigation 值得借鉴的地方是: + +1. Android 手机可以承担上位机主脑,直接跑 TFLite 策略并下发左右轮命令。 +2. ARCore 可以提供短距离相对位姿,但它依赖 Google ARCore 运行环境,国内实机可用性不稳定。 +3. 端到端策略网络可以把图像和目标向量直接映射到差速控制,但这要求训练数据和真实场景分布足够接近。 +4. 当前实现安全兜底较薄,不适合作为购物车首版安全策略直接照搬。 + +如果本项目首版继续保持“OpenBot + Android 手机 + MCU + 差速底盘”的主线,更现实的参考方式是: + +- 复用它的“Android 端模型推理 -> `Vehicle.setControl()` -> 下位机串口命令”链路。 +- 不依赖 ARCore 作为核心能力,因为目标手机环境可能无法稳定安装和运行 Google Play Services for AR。 +- 对购物车跟随主线,应继续采用已有的目标检测、目标锁定、ReID 辅助、距离控制和安全状态机,而不是把 PointGoalNavigation 当作可直接替换的导航模块。 + +## 简化伪代码 + +```java +onStartPointGoalNavigation() { + askUserForForwardAndLeftMeters(); + requireCameraPermission(); + startArCoreSession(); +} + +startDriving(forward, left) { + goalX = -left; // ARCore x positive means right + goalZ = -forward; // ARCore z positive means backward + + startAnchor = currentArCorePose; + targetAnchor = startAnchor.compose(translation(goalX, 0, goalZ)); + navigationPolicy = load("networks/navigation.tflite"); + isRunning = true; +} + +onArCoreUpdate(currentPose, targetPose, cameraImage) { + if (!isRunning) return; + + distance = horizontalDistance(currentPose, targetPose); + if (distance < 0.15) { + stop(); + showGoalReached(); + return; + } + + deltaYaw = yawDifference(currentPose.forward, targetPose - currentPose); + image = yuvToRgbResizeAndCrop(cameraImage, 160, 90); + + control = navigationPolicy.run( + image, + distance, + sin(deltaYaw), + cos(deltaYaw)); + + vehicle.setControl(control.left, control.right); +} + +onArCoreTrackingLostOrPaused() { + stop(); + showFailureMessage(); +} +``` + diff --git a/android/robot/src/main/java/org/openbot/cartfollow/ActionArbitrator.java b/android/robot/src/main/java/org/openbot/cartfollow/ActionArbitrator.java index 0d61b338e..a95ad7dac 100644 --- a/android/robot/src/main/java/org/openbot/cartfollow/ActionArbitrator.java +++ b/android/robot/src/main/java/org/openbot/cartfollow/ActionArbitrator.java @@ -23,7 +23,15 @@ public BehaviorDecisionResult decide( } if (state == FollowState.REACQUIRE_TARGET) { float conf = identity == null ? 0f : identity.confidence; - return result(state, BehaviorAction.REACQUIRE_HOLD, "reacquire_confirming", null, conf); + String reason = + identity != null + && identity.looseAdmissionOk() + && !identity.bboxDefaultOk() + && !identity.predictionOk() + && identity.trackId != identity.lockedTrackId + ? "motion_gate_failed" + : "reacquire_confirming"; + return result(state, BehaviorAction.REACQUIRE_HOLD, reason, null, conf); } if (state == FollowState.IDENTITY_UNCERTAIN) { float conf = identity == null ? 0f : identity.confidence; diff --git a/android/robot/src/main/java/org/openbot/cartfollow/BboxContinuityEvidence.java b/android/robot/src/main/java/org/openbot/cartfollow/BboxContinuityEvidence.java index 219600f55..b8104ac72 100644 --- a/android/robot/src/main/java/org/openbot/cartfollow/BboxContinuityEvidence.java +++ b/android/robot/src/main/java/org/openbot/cartfollow/BboxContinuityEvidence.java @@ -11,11 +11,16 @@ public class BboxContinuityEvidence { public static final float STRICT_X_MAX = 0.18f; public static final float STRICT_AREA_MIN = 0.60f; public static final float STRICT_AREA_MAX = 1.67f; + public static final float LOOSE_CENTER_MAX = 0.36f; + public static final float LOOSE_X_MAX = 0.36f; + public static final float LOOSE_AREA_MIN = 0.35f; + public static final float LOOSE_AREA_MAX = 2.80f; public final float centerJumpRatio; public final float xJumpRatio; public final float areaRatio; public final float predictionError; + public final boolean looseAdmissionOk; public final boolean bboxDefaultOk; public final boolean bboxStrictOk; public final boolean predictionOk; @@ -31,6 +36,11 @@ public BboxContinuityEvidence( this.xJumpRatio = xJumpRatio; this.areaRatio = areaRatio; this.predictionError = predictionError; + this.looseAdmissionOk = + centerJumpRatio <= LOOSE_CENTER_MAX + && xJumpRatio <= LOOSE_X_MAX + && areaRatio >= LOOSE_AREA_MIN + && areaRatio <= LOOSE_AREA_MAX; this.bboxDefaultOk = centerJumpRatio <= DEFAULT_CENTER_MAX && xJumpRatio <= DEFAULT_X_MAX diff --git a/android/robot/src/main/java/org/openbot/cartfollow/FollowStateMachine.java b/android/robot/src/main/java/org/openbot/cartfollow/FollowStateMachine.java index 004938b48..01f059048 100644 --- a/android/robot/src/main/java/org/openbot/cartfollow/FollowStateMachine.java +++ b/android/robot/src/main/java/org/openbot/cartfollow/FollowStateMachine.java @@ -335,7 +335,8 @@ public FrameResult onFrame( updateEvidenceCounters(id, m, safePersons); Recognition selected = selectedCandidate(id, m); if (selected != null - && (strongStrictStreak >= UNCERTAIN_RECOVER_FRAMES + && (identityAdmissionReady(id) + || strongStrictStreak >= UNCERTAIN_RECOVER_FRAMES || midDefaultStreak >= UNCERTAIN_RECOVER_FRAMES)) { state = FollowState.REACQUIRE_TARGET; matchCount = 0; @@ -433,7 +434,7 @@ private boolean followConfident( if (id != null && id.hasBelief()) { return id.beliefConfirmed() && id.beliefStableFrames >= 2 - && (id.bboxDefaultOk() || id.predictionOk() || id.trackId == id.lockedTrackId); + && motionGateOk(id); } if (id != null && id.reidAvailable()) { return id.weakOk() && id.bboxDefaultOk(); @@ -444,8 +445,7 @@ private boolean followConfident( private boolean followCaution( IdentityEvidence id, TargetMatcher.MatchResult m, List persons) { if (id != null && id.hasBelief()) { - return id.beliefCaution() - && (id.bboxDefaultOk() || id.predictionOk() || id.trackId == id.lockedTrackId); + return id.beliefCaution() && motionGateOk(id); } if (id != null && id.reidAvailable()) { return id.bboxDefaultOk() && (id.weakOk() || id.midOk()); @@ -458,7 +458,7 @@ private boolean reacquireReady( if (id != null && id.hasBelief()) { return id.beliefConfirmed() && id.beliefStableFrames >= 3 - && (id.bboxDefaultOk() || id.predictionOk() || id.trackId == id.lockedTrackId); + && motionGateOk(id); } if (id != null && id.reidAvailable()) { return (id.strongOk() && id.bboxDefaultOk()) || (id.midOk() && id.bboxStrictOk()); @@ -469,7 +469,7 @@ private boolean reacquireReady( private boolean lostRecoverReady( IdentityEvidence id, TargetMatcher.MatchResult m, List persons) { if (id != null && id.hasBelief()) { - return id.beliefConfirmed() && id.beliefStableFrames >= LOST_RECOVER_FRAMES; + return identityAdmissionReady(id); } if (id != null && id.reidAvailable()) { return strongStrictStreak >= LOST_RECOVER_FRAMES @@ -488,7 +488,7 @@ private boolean lostRecoverReady( private void updateEvidenceCounters( IdentityEvidence id, TargetMatcher.MatchResult m, List persons) { if (id != null && id.hasBelief()) { - boolean safeBbox = id.bboxDefaultOk() || id.predictionOk() || id.trackId == id.lockedTrackId; + boolean safeBbox = motionGateOk(id); boolean midDefault = id.targetBelief >= IdentityBelief.BELIEF_CAUTION && safeBbox; boolean strongDefault = id.targetBelief >= IdentityBelief.BELIEF_CONFIRM && safeBbox; boolean strongStrict = @@ -521,6 +521,18 @@ private void updateEvidenceCounters( if (midDefault || strongDefault || strongStrict) unstableStreak = 0; } + private boolean identityAdmissionReady(IdentityEvidence id) { + if (id == null || !id.hasBelief()) return false; + boolean admission = + id.looseAdmissionOk() || id.bboxDefaultOk() || id.predictionOk() || id.trackId == id.lockedTrackId; + return id.beliefCaution() && id.beliefStableFrames >= UNCERTAIN_RECOVER_FRAMES && admission; + } + + private boolean motionGateOk(IdentityEvidence id) { + if (id == null) return false; + return id.bboxDefaultOk() || id.predictionOk() || id.trackId == id.lockedTrackId; + } + private void resetEvidenceCounters() { midDefaultStreak = 0; strongDefaultStreak = 0; diff --git a/android/robot/src/main/java/org/openbot/cartfollow/HumanCartSimulatorFragment.java b/android/robot/src/main/java/org/openbot/cartfollow/HumanCartSimulatorFragment.java index e8fbbba98..58c509a4b 100644 --- a/android/robot/src/main/java/org/openbot/cartfollow/HumanCartSimulatorFragment.java +++ b/android/robot/src/main/java/org/openbot/cartfollow/HumanCartSimulatorFragment.java @@ -45,6 +45,7 @@ public class HumanCartSimulatorFragment extends CameraFragment { private static final int COLOR_CANDIDATE = 1; private static final int COLOR_NORMAL = 2; private static final int COLOR_FAIL = 3; + private static final int RECOVERY_RELOCK_MIN_FRAMES = 2; private FragmentHumanCartSimulatorBinding binding; private Handler handler; @@ -79,12 +80,15 @@ public class HumanCartSimulatorFragment extends CameraFragment { private final CartFollowDiagnosticConfig diagnosticConfig = new CartFollowDiagnosticConfig(); private final CartFollowDiagnosticSaver diagnosticSaver = new CartFollowDiagnosticSaver(); private CartFollowDiagnosticSession diagnosticSession; + private boolean diagnosticEnabled = false; private boolean diagnosticActive = false; private boolean targetEventAwaitingReturn = false; private boolean showFullDebug = false; private long lastDiagnosticFrameLogMs = 0L; private long lastDiagnosticCropMs = 0L; private long lastDiagnosticGalleryMs = 0L; + private int recoveryRelockTrackId = -1; + private int recoveryRelockFrames = 0; private Bitmap latestConfirmSnapshot; private final List drawBoxes = new ArrayList<>(); @@ -162,6 +166,15 @@ public void onViewCreated(@NonNull View view, @Nullable Bundle savedInstanceStat }); resetTargetEventButton(); binding.btnTargetEvent.setOnClickListener(v -> recordTargetEvent()); + binding.diagnosticSwitch.setChecked(false); + binding.diagnosticSwitch.setOnClickListener( + v -> { + diagnosticEnabled = binding.diagnosticSwitch.isChecked(); + if (!diagnosticEnabled) { + stopDiagnosticSession(); + } + resetTargetEventButton(); + }); binding.btnConfirm.setOnClickListener( v -> { @@ -169,7 +182,7 @@ public void onViewCreated(@NonNull View view, @Nullable Bundle savedInstanceStat int lockedTrackId = targetTrackManager.lockClosest(stateMachine.getMemory().getLastBbox()); beliefAccumulator.lockTrack(lockedTrackId); activateDiagnosticSession(); - if (latestConfirmSnapshot != null) { + if (diagnosticActive && diagnosticSession != null && latestConfirmSnapshot != null) { diagnosticSaver.saveGallerySnapshotAsync( latestConfirmSnapshot, diagnosticSession, "confirmed_snapshot"); } @@ -180,6 +193,7 @@ public void onViewCreated(@NonNull View view, @Nullable Bundle savedInstanceStat if (reidCoordinator != null) reidCoordinator.reset(); targetTrackManager.reset(); beliefAccumulator.reset(); + resetRecoveryRelock(); stopDiagnosticSession(); startDiagnosticSession(); stateMachine.retake(); @@ -189,6 +203,7 @@ public void onViewCreated(@NonNull View view, @Nullable Bundle savedInstanceStat if (reidCoordinator != null) reidCoordinator.reset(); targetTrackManager.reset(); beliefAccumulator.reset(); + resetRecoveryRelock(); stopDiagnosticSession(); stateMachine.cancel(); }); @@ -201,6 +216,7 @@ public void onViewCreated(@NonNull View view, @Nullable Bundle savedInstanceStat if (reidCoordinator != null) reidCoordinator.reset(); targetTrackManager.reset(); beliefAccumulator.reset(); + resetRecoveryRelock(); startDiagnosticSession(); stateMachine.startCapture(); } else { @@ -208,6 +224,7 @@ public void onViewCreated(@NonNull View view, @Nullable Bundle savedInstanceStat if (reidCoordinator != null) reidCoordinator.reset(); targetTrackManager.reset(); beliefAccumulator.reset(); + resetRecoveryRelock(); stopDiagnosticSession(); stateMachine.cancel(); resetUiToIdle(); @@ -410,6 +427,7 @@ protected void processFrame(Bitmap bitmap, ImageProxy image) { stateMachine.onFrame( mappedRecognitions, workingFrame, frameW, frameH, sensorOrientation, identity); fr.behaviorDecision = decideBehavior(fr, frameW, frameH); + maybeRelockAfterRecovery(fr); updateDrawState(fr, frameW, frameH, sensorOrientation); String commandText = commandForState(fr); @@ -436,6 +454,70 @@ private void updateCropImageInfo() { recreateNetwork(getModel(), getDevice(), getNumThreads()); } + private void maybeRelockAfterRecovery(FollowStateMachine.FrameResult fr) { + if (fr == null || fr.identityEvidence == null || fr.behaviorDecision == null) { + resetRecoveryRelock(); + return; + } + IdentityEvidence identity = fr.identityEvidence; + if (!isRelockState(fr.state) + || !isRelockAction(fr.behaviorDecision.selectedAction) + || identity.trackId < 0 + || identity.trackId == identity.lockedTrackId + || !passesRelockMotionGate(identity)) { + resetRecoveryRelock(); + return; + } + + if (recoveryRelockTrackId != identity.trackId) { + recoveryRelockTrackId = identity.trackId; + recoveryRelockFrames = 1; + return; + } + recoveryRelockFrames++; + if (recoveryRelockFrames < RECOVERY_RELOCK_MIN_FRAMES) return; + + if (targetTrackManager.lockTrack(identity.trackId, "relock_after_recovery")) { + beliefAccumulator.lockTrack(identity.trackId); + fr.behaviorDecision = + new BehaviorDecisionResult( + fr.behaviorDecision.state, + fr.behaviorDecision.selectedAction, + appendActionReason(fr.behaviorDecision.actionReason, "relock_after_recovery"), + fr.behaviorDecision.safetyBlockReason, + fr.behaviorDecision.confidence, + fr.behaviorDecision.distanceEvidence, + fr.behaviorDecision.traversabilityEvidence); + } + resetRecoveryRelock(); + } + + private static boolean isRelockState(FollowState state) { + return state == FollowState.REACQUIRE_TARGET + || state == FollowState.READY_TO_FOLLOW + || state == FollowState.FOLLOW_CAUTION + || state == FollowState.FOLLOW; + } + + private static boolean isRelockAction(BehaviorAction action) { + return action == BehaviorAction.FOLLOW_CAUTION || action == BehaviorAction.FOLLOW_SLOW; + } + + private static boolean passesRelockMotionGate(IdentityEvidence identity) { + return identity.bboxDefaultOk() || identity.predictionOk(); + } + + private void resetRecoveryRelock() { + recoveryRelockTrackId = -1; + recoveryRelockFrames = 0; + } + + private static String appendActionReason(String reason, String addition) { + if (reason == null || reason.isEmpty()) return addition; + if (reason.contains(addition)) return reason; + return reason + "|" + addition; + } + private synchronized void updateDrawState( FollowStateMachine.FrameResult fr, int frameW, int frameH, int sensorOrientation) { drawBoxes.clear(); @@ -597,6 +679,10 @@ private void updateUiForState(FollowStateMachine.FrameResult fr) { private void startDiagnosticSession() { stopDiagnosticSession(); + if (!diagnosticEnabled) { + resetDiagnosticState(); + return; + } diagnosticSession = new CartFollowDiagnosticSession(requireContext().getApplicationContext()); diagnosticSession.initCsvFiles(); diagnosticActive = false; @@ -609,9 +695,14 @@ private void startDiagnosticSession() { } private void activateDiagnosticSession() { + if (!diagnosticEnabled) { + resetDiagnosticState(); + return; + } if (diagnosticSession == null) { startDiagnosticSession(); } + if (diagnosticSession == null) return; diagnosticActive = true; targetEventAwaitingReturn = false; String detectorName = getModel() == null ? "" : getModel().name; @@ -634,9 +725,13 @@ private void activateDiagnosticSession() { } private void stopDiagnosticSession() { - if (diagnosticSession != null && diagnosticActive) { + if (diagnosticEnabled && diagnosticSession != null && diagnosticActive) { diagnosticSaver.saveEventAsync(diagnosticSession, frameNum, "session_stop", ""); } + resetDiagnosticState(); + } + + private void resetDiagnosticState() { diagnosticActive = false; diagnosticSession = null; targetEventAwaitingReturn = false; @@ -653,14 +748,14 @@ private void resetTargetEventButton() { activity.runOnUiThread( () -> { if (binding == null) return; - binding.btnTargetEvent.setEnabled(diagnosticActive); + binding.btnTargetEvent.setEnabled(diagnosticEnabled && diagnosticActive); binding.btnTargetEvent.setText( targetEventAwaitingReturn ? "目标回到画面" : "目标离开画面"); }); } private void recordTargetEvent() { - if (!diagnosticActive || diagnosticSession == null) return; + if (!diagnosticEnabled || !diagnosticActive || diagnosticSession == null) return; String eventType = targetEventAwaitingReturn ? "target_return" : "target_left"; diagnosticSaver.saveEventAsync(diagnosticSession, frameNum, eventType, ""); targetEventAwaitingReturn = !targetEventAwaitingReturn; @@ -675,7 +770,7 @@ private void maybeSaveDiagnostics( int frameW, int frameH, int sensorOrientation) { - if (!diagnosticActive || diagnosticSession == null || fr == null) return; + if (!diagnosticEnabled || !diagnosticActive || diagnosticSession == null || fr == null) return; long now = SystemClock.elapsedRealtime(); boolean shouldLog = lastDiagnosticFrameLogMs == 0L @@ -713,7 +808,7 @@ private void maybeSaveDiagnostics( private void maybeSaveGalleryCandidate( Bitmap frame, Detector.Recognition candidate, int sensorOrientation) { - if (diagnosticSession == null || frame == null || candidate == null) return; + if (!diagnosticEnabled || diagnosticSession == null || frame == null || candidate == null) return; if (candidate.getLocation() == null) return; long now = SystemClock.elapsedRealtime(); if (lastDiagnosticGalleryMs != 0L @@ -853,7 +948,7 @@ private void updateDebugInfo( private String buildIdentityDebugLine(IdentityEvidence identity) { if (identity == null) { - return "reidAvailable=false\nreidCrop=upright\ngallerySize=0\nbestScore=0.000\nsecondScore=0.000\nmargin=0.000\nweak/mid/strong=false/false/false\nbboxDefault=false bboxStrict=false prediction=false\nstableMatchCount=0\ncandidateSwitchCount=0\nreidLatencyMs=0\nreidReason=-\nactiveTrackCount=0\ntrackId=-1 lockedTrackId=-1 suspectedTrackId=-1\ntrackAge=0 missedFrames=0\nbelief=0.00 beliefStable=0 beliefUncertain=0\nbeliefReason=-"; + return "reidAvailable=false\nreidCrop=upright\ngallerySize=0\nbestScore=0.000\nsecondScore=0.000\nmargin=0.000\nweak/mid/strong=false/false/false\nbboxLoose=false bboxDefault=false bboxStrict=false prediction=false\nstableMatchCount=0\ncandidateSwitchCount=0\nreidLatencyMs=0\nreidReason=-\nactiveTrackCount=0\ntrackId=-1 lockedTrackId=-1 suspectedTrackId=-1\ntrackAge=0 missedFrames=0\nbelief=0.00 beliefStable=0 beliefUncertain=0\nbeliefReason=-"; } ReIDMatchResult reid = identity.reidMatch; BboxContinuityEvidence bbox = identity.bboxContinuity; @@ -866,7 +961,7 @@ private String buildIdentityDebugLine(IdentityEvidence identity) { String reason = reid == null ? identity.reason : reid.reason; return String.format( Locale.US, - "reidAvailable=%s\nreidCrop=upright\ngallerySize=%d\nbestScore=%.3f\nsecondScore=%.3f\nmargin=%.3f\nweak/mid/strong=%s/%s/%s\nbboxDefault=%s bboxStrict=%s prediction=%s\nstableMatchCount=%d\ncandidateSwitchCount=%d\nreidLatencyMs=%d\nreidReason=%s\nactiveTrackCount=%d\ntrackId=%d lockedTrackId=%d suspectedTrackId=%d\ntrackAge=%d missedFrames=%d\nbelief=%.2f reidC=%.2f bboxC=%.2f predC=%.2f switchP=%.2f\nbeliefStable=%d beliefUncertain=%d\nbeliefReason=%s", + "reidAvailable=%s\nreidCrop=upright\ngallerySize=%d\nbestScore=%.3f\nsecondScore=%.3f\nmargin=%.3f\nweak/mid/strong=%s/%s/%s\nbboxLoose=%s bboxDefault=%s bboxStrict=%s prediction=%s\nstableMatchCount=%d\ncandidateSwitchCount=%d\nreidLatencyMs=%d\nreidReason=%s\nactiveTrackCount=%d\ntrackId=%d lockedTrackId=%d suspectedTrackId=%d\ntrackAge=%d missedFrames=%d\nbelief=%.2f reidC=%.2f bboxC=%.2f predC=%.2f switchP=%.2f\nbeliefStable=%d beliefUncertain=%d\nbeliefReason=%s", reidAvailable, gallerySize, best, @@ -875,6 +970,7 @@ private String buildIdentityDebugLine(IdentityEvidence identity) { identity.weakOk(), identity.midOk(), identity.strongOk(), + bbox != null && bbox.looseAdmissionOk, bbox != null && bbox.bboxDefaultOk, bbox != null && bbox.bboxStrictOk, bbox != null && bbox.predictionOk, diff --git a/android/robot/src/main/java/org/openbot/cartfollow/IdentityBeliefAccumulator.java b/android/robot/src/main/java/org/openbot/cartfollow/IdentityBeliefAccumulator.java index d2b1109a8..301b0ef7f 100644 --- a/android/robot/src/main/java/org/openbot/cartfollow/IdentityBeliefAccumulator.java +++ b/android/robot/src/main/java/org/openbot/cartfollow/IdentityBeliefAccumulator.java @@ -1,5 +1,6 @@ package org.openbot.cartfollow; +import android.os.SystemClock; import java.util.HashMap; import java.util.List; import java.util.Locale; @@ -7,6 +8,8 @@ import org.openbot.tflite.Detector.Recognition; public class IdentityBeliefAccumulator { + private static final float NO_SPATIAL_SUPPORT_BELIEF_CAP = IdentityBelief.BELIEF_CAUTION - 0.05f; + private final Map beliefs = new HashMap<>(); private final Map stableFrames = new HashMap<>(); private final Map uncertainFrames = new HashMap<>(); @@ -65,7 +68,7 @@ public IdentityEvidence update( for (TargetTrack track : tracks) { IdentityBelief belief = - updateTrackBelief(base, track, reidCandidateTrack, lockedTrack, memory, frameW, frameH); + updateTrackBelief(base, track, trackManager, reidCandidateTrack, lockedTrack, memory, frameW, frameH); BboxContinuityEvidence bbox = bboxEvidence(track, memory, frameW, frameH); if (!track.isVisible()) continue; if (selectedTrack == null || shouldSelect(track, belief, selectedTrack, selectedBelief)) { @@ -93,8 +96,53 @@ && getBeliefForTrack(lockedTrack) >= IdentityBelief.BELIEF_LOST) { selectedBbox = bboxEvidence(lockedTrack, memory, frameW, frameH); } + if (selectedTrack != null + && selectedBelief != null + && selectedTrack.trackId != managerLockedId + && selectedBelief.targetBelief >= IdentityBelief.BELIEF_CAUTION + && hasSpatialSupport(selectedTrack, selectedBbox, trackManager, frameW, frameH)) { + boolean accepted = + trackManager.updateSuspectedTrack(selectedTrack.trackId, selectedBelief.targetBelief); + if (!accepted) { + TargetTrack held = trackManager.getTrackById(trackManager.getSuspectedTrackId()); + if (held != null && held.isVisible()) { + selectedTrack = held; + selectedBbox = bboxEvidence(held, memory, frameW, frameH); + selectedBelief = + new IdentityBelief( + held.trackId, + getBeliefForTrack(held), + 0f, + 0f, + 0f, + 0f, + getStableFrames(held.trackId), + getUncertainFrames(held.trackId), + "suspected_dwell_hold"); + } + } + } else if (selectedTrack != null + && selectedBelief != null + && selectedTrack.trackId != managerLockedId + && selectedBelief.targetBelief >= IdentityBelief.BELIEF_LOST + && !hasSpatialSupport(selectedTrack, selectedBbox, trackManager, frameW, frameH)) { + selectedBelief = + new IdentityBelief( + selectedBelief.trackId, + Math.min(selectedBelief.targetBelief, NO_SPATIAL_SUPPORT_BELIEF_CAP), + selectedBelief.reidContribution, + selectedBelief.bboxContribution, + selectedBelief.predictionContribution, + selectedBelief.switchPenalty, + selectedBelief.stableFrames, + selectedBelief.uncertainFrames, + appendReason( + selectedBelief.beliefReason, + "reid_interest_no_spatial_support spatial_support_missing")); + } + if (selectedTrack == null || selectedBelief == null) { - trackManager.setSuspectedTrackId(-1); + trackManager.updateSuspectedTrack(-1, 0f); return withBelief( base, null, @@ -107,7 +155,10 @@ && getBeliefForTrack(lockedTrack) >= IdentityBelief.BELIEF_LOST) { "no_visible_track"); } - if (selectedTrack.trackId != lastSelectedTrackId) { + boolean countableSelection = + selectedTrack.trackId == managerLockedId + || hasSpatialSupport(selectedTrack, selectedBbox, trackManager, frameW, frameH); + if (countableSelection && selectedTrack.trackId != lastSelectedTrackId) { if (lastSelectedTrackId >= 0) candidateSwitchCount++; lastSelectedTrackId = selectedTrack.trackId; } @@ -115,9 +166,9 @@ && getBeliefForTrack(lockedTrack) >= IdentityBelief.BELIEF_LOST) { int suspectedTrackId = -1; if (selectedTrack.trackId != managerLockedId && selectedBelief.targetBelief >= IdentityBelief.BELIEF_CAUTION) { - suspectedTrackId = selectedTrack.trackId; + suspectedTrackId = trackManager.getSuspectedTrackId(); } - trackManager.setSuspectedTrackId(suspectedTrackId); + if (suspectedTrackId < 0) trackManager.updateSuspectedTrack(-1, 0f); return withBelief( base, @@ -134,6 +185,7 @@ && getBeliefForTrack(lockedTrack) >= IdentityBelief.BELIEF_LOST) { private IdentityBelief updateTrackBelief( IdentityEvidence base, TargetTrack track, + TargetTrackManager trackManager, TargetTrack reidCandidateTrack, TargetTrack lockedTrack, TargetMemory memory, @@ -145,12 +197,24 @@ private IdentityBelief updateTrackBelief( boolean isLocked = track.trackId == lockedTrackId; boolean lockedVisible = lockedTrack != null && lockedTrack.isVisible(); boolean isReidCandidate = reidCandidateTrack != null && reidCandidateTrack.trackId == track.trackId; + long now = SystemClock.elapsedRealtime(); + boolean nearLockedGhost = + trackManager != null && trackManager.isNearLockedGhost(track, frameW, frameH, now); BboxContinuityEvidence bbox = bboxEvidence(track, memory, frameW, frameH); - float reidContribution = reidContribution(base, isReidCandidate); + boolean spatialSupport = hasSpatialSupport(track, bbox, trackManager, frameW, frameH); + boolean missingSpatialSupport = !isLocked && !spatialSupport; + float rawReidContribution = reidContribution(base, isReidCandidate); + float reidContribution = + missingSpatialSupport ? Math.min(rawReidContribution, 0.04f) : rawReidContribution; float bboxContribution = bbox.bboxStrictOk ? 0.15f : (bbox.bboxDefaultOk ? 0.10f : 0f); float predictionContribution = bbox.predictionOk ? 0.06f : 0f; + float looseAdmissionContribution = + !missingSpatialSupport && !bbox.bboxDefaultOk && bbox.looseAdmissionOk && isReidCandidate + ? 0.04f + : 0f; float lockedContribution = isLocked && track.isVisible() ? 0.08f : 0f; + float ghostContribution = !lockedVisible && nearLockedGhost && track.isVisible() ? 0.06f : 0f; float ageContribution = track.ageFrames >= 3 && track.isVisible() ? 0.04f : 0f; float switchPenalty = isReidCandidate && lastSelectedTrackId >= 0 && track.trackId != lastSelectedTrackId ? 0.12f : 0f; @@ -159,16 +223,27 @@ private IdentityBelief updateTrackBelief( float missedPenalty = Math.min(0.30f, track.missedFrames * 0.08f); float weakMarginPenalty = isReidCandidate && base != null && base.reidAvailable() && !base.weakOk() ? 0.08f : 0f; + float spatialSupportPenalty = missingSpatialSupport && isReidCandidate ? 0.06f : 0f; float decay = track.isVisible() ? 0.80f : 0.70f; float positive = reidContribution + bboxContribution + predictionContribution + + looseAdmissionContribution + lockedContribution + + ghostContribution + ageContribution; - float penalty = switchPenalty + lockedProtectionPenalty + missedPenalty + weakMarginPenalty; + float penalty = + switchPenalty + + lockedProtectionPenalty + + missedPenalty + + weakMarginPenalty + + spatialSupportPenalty; float belief = clamp(old * decay + positive - penalty, 0f, 1f); + if (missingSpatialSupport && belief >= IdentityBelief.BELIEF_CAUTION) { + belief = Math.min(belief, NO_SPATIAL_SUPPORT_BELIEF_CAP); + } int stable = belief >= IdentityBelief.BELIEF_CAUTION ? getStableFrames(track.trackId) + 1 : 0; int uncertain = @@ -180,15 +255,22 @@ private IdentityBelief updateTrackBelief( String reason = String.format( Locale.US, - "belief=%.2f old=%.2f reid=%.2f bbox=%.2f pred=%.2f locked=%s switchPenalty=%.2f missed=%d", + "belief=%.2f old=%.2f reid=%.2f bbox=%.2f pred=%.2f loose=%.2f ghost=%.2f locked=%s switchPenalty=%.2f missed=%d%s%s%s", belief, old, reidContribution, bboxContribution, predictionContribution, + looseAdmissionContribution, + ghostContribution, isLocked, switchPenalty + lockedProtectionPenalty, - track.missedFrames); + track.missedFrames, + looseAdmissionContribution > 0f ? " loose_admission_only" : "", + ghostContribution > 0f ? " locked_ghost_reference" : "", + missingSpatialSupport && isReidCandidate + ? " reid_interest_no_spatial_support spatial_support_missing" + : ""); return new IdentityBelief( track.trackId, belief, @@ -265,6 +347,20 @@ private boolean shouldSelect( return track.ageFrames > selectedTrack.ageFrames; } + private static boolean hasSpatialSupport( + TargetTrack track, + BboxContinuityEvidence bbox, + TargetTrackManager trackManager, + int frameW, + int frameH) { + if (track == null) return false; + if (bbox != null && (bbox.looseAdmissionOk || bbox.bboxDefaultOk || bbox.predictionOk)) { + return true; + } + return trackManager != null + && trackManager.isNearLockedGhost(track, frameW, frameH, SystemClock.elapsedRealtime()); + } + private static BboxContinuityEvidence bboxEvidence( TargetTrack track, TargetMemory memory, int frameW, int frameH) { if (track == null || memory == null || track.lastBbox == null) { @@ -282,6 +378,13 @@ private static float reidContribution(IdentityEvidence base, boolean isReidCandi return base.matched ? 0.05f : 0f; } + private static String appendReason(String reason, String addition) { + if (addition == null || addition.isEmpty()) return reason; + if (reason == null || reason.isEmpty()) return addition; + if (reason.contains(addition)) return reason; + return reason + " " + addition; + } + private int getStableFrames(int trackId) { Integer value = stableFrames.get(trackId); return value == null ? 0 : value; diff --git a/android/robot/src/main/java/org/openbot/cartfollow/IdentityEvidence.java b/android/robot/src/main/java/org/openbot/cartfollow/IdentityEvidence.java index 206cc79e9..296f5306a 100644 --- a/android/robot/src/main/java/org/openbot/cartfollow/IdentityEvidence.java +++ b/android/robot/src/main/java/org/openbot/cartfollow/IdentityEvidence.java @@ -152,6 +152,10 @@ public boolean bboxStrictOk() { return bboxContinuity != null && bboxContinuity.bboxStrictOk; } + public boolean looseAdmissionOk() { + return bboxContinuity != null && bboxContinuity.looseAdmissionOk; + } + public boolean predictionOk() { return bboxContinuity != null && bboxContinuity.predictionOk; } diff --git a/android/robot/src/main/java/org/openbot/cartfollow/TargetTrackManager.java b/android/robot/src/main/java/org/openbot/cartfollow/TargetTrackManager.java index cd4a5edd8..44cb28a14 100644 --- a/android/robot/src/main/java/org/openbot/cartfollow/TargetTrackManager.java +++ b/android/robot/src/main/java/org/openbot/cartfollow/TargetTrackManager.java @@ -12,18 +12,42 @@ public class TargetTrackManager { private static final float AREA_RATIO_MIN = 0.50f; private static final float AREA_RATIO_MAX = 2.00f; private static final float IOU_SOFT_GATE = 0.10f; - private static final int MAX_MISSED_FRAMES = 12; + private static final int MAX_MISSED_FRAMES = 16; + private static final long LOCKED_GHOST_TTL_MS = 3000L; + private static final int SUSPECTED_MIN_DWELL_FRAMES = 3; + private static final int SUSPECTED_REPLACEMENT_STABLE_FRAMES = 2; + private static final float SWITCH_BELIEF_DELTA = 0.15f; private final List tracks = new ArrayList<>(); private int nextTrackId = 1; private int lockedTrackId = -1; private int suspectedTrackId = -1; + private int suspectedDwellFrames = 0; + private float suspectedBelief = 0f; + private String suspectedUpdateReason = "none"; + private RectF lockedGhostBbox = null; + private float lockedGhostCenterX = 0f; + private float lockedGhostCenterY = 0f; + private float lockedGhostVelocityX = 0f; + private float lockedGhostVelocityY = 0f; + private long lockedGhostLastSeenMs = 0L; + private String lockedGhostLostSide = "unknown"; public void reset() { tracks.clear(); nextTrackId = 1; lockedTrackId = -1; suspectedTrackId = -1; + suspectedDwellFrames = 0; + suspectedBelief = 0f; + suspectedUpdateReason = "reset"; + lockedGhostBbox = null; + lockedGhostCenterX = 0f; + lockedGhostCenterY = 0f; + lockedGhostVelocityX = 0f; + lockedGhostVelocityY = 0f; + lockedGhostLastSeenMs = 0L; + lockedGhostLostSide = "unknown"; } public void update(List detections, int frameW, int frameH, long timestampMs) { @@ -58,6 +82,11 @@ public void update(List detections, int frameW, int frameH, long ti } tracks.add(new TargetTrack(nextTrackId++, detection, timestampMs)); } + + TargetTrack lockedTrack = getLockedTrack(); + if (lockedTrack != null && lockedTrack.isVisible()) { + updateLockedGhost(lockedTrack, frameW, timestampMs); + } } public List getTracks() { @@ -102,6 +131,80 @@ public int getSuspectedTrackId() { public void setSuspectedTrackId(int suspectedTrackId) { this.suspectedTrackId = suspectedTrackId; + this.suspectedDwellFrames = suspectedTrackId < 0 ? 0 : 1; + this.suspectedBelief = 0f; + this.suspectedUpdateReason = "forced"; + } + + public int getSuspectedDwellFrames() { + return suspectedDwellFrames; + } + + public String getSuspectedUpdateReason() { + return suspectedUpdateReason; + } + + public boolean updateSuspectedTrack(int candidateTrackId, float candidateBelief) { + if (candidateTrackId < 0) { + suspectedTrackId = -1; + suspectedDwellFrames = 0; + suspectedBelief = 0f; + suspectedUpdateReason = "clear"; + return true; + } + if (candidateTrackId == suspectedTrackId) { + suspectedDwellFrames++; + suspectedBelief = Math.max(suspectedBelief, candidateBelief); + suspectedUpdateReason = "suspected_dwell_keep"; + return true; + } + TargetTrack candidate = getTrackById(candidateTrackId); + TargetTrack current = getTrackById(suspectedTrackId); + if (suspectedTrackId < 0 || current == null) { + suspectedTrackId = candidateTrackId; + suspectedDwellFrames = 1; + suspectedBelief = candidateBelief; + suspectedUpdateReason = "suspected_init"; + return true; + } + boolean candidateStable = + candidate != null && candidate.isVisible() && candidate.stableFrames >= SUSPECTED_REPLACEMENT_STABLE_FRAMES; + boolean clearlyBetter = candidateBelief >= suspectedBelief + SWITCH_BELIEF_DELTA; + boolean currentWeak = !current.isVisible() || current.missedFrames > 0; + boolean dwellSatisfied = suspectedDwellFrames >= SUSPECTED_MIN_DWELL_FRAMES; + if (candidateStable && clearlyBetter && (currentWeak || dwellSatisfied)) { + suspectedTrackId = candidateTrackId; + suspectedDwellFrames = 1; + suspectedBelief = candidateBelief; + suspectedUpdateReason = "suspected_switch_clear_better"; + return true; + } + suspectedDwellFrames++; + suspectedUpdateReason = "suspected_dwell_hold"; + return false; + } + + public boolean hasLockedGhost(long nowMs) { + return lockedGhostBbox != null && nowMs - lockedGhostLastSeenMs <= LOCKED_GHOST_TTL_MS; + } + + public RectF getLockedGhostBbox(long nowMs) { + return hasLockedGhost(nowMs) ? new RectF(lockedGhostBbox) : null; + } + + public String getLockedGhostLostSide(long nowMs) { + return hasLockedGhost(nowMs) ? lockedGhostLostSide : "unknown"; + } + + public boolean isNearLockedGhost(TargetTrack track, int frameW, int frameH, long nowMs) { + if (track == null || track.lastBbox == null || !hasLockedGhost(nowMs)) return false; + RectF ghost = predictedLockedGhost(nowMs); + float diag = (float) Math.hypot(Math.max(1, frameW), Math.max(1, frameH)); + float centerJump = centerDistance(track.lastBbox, ghost) / Math.max(1f, diag); + float areaRatio = area(track.lastBbox) / Math.max(1f, area(ghost)); + return centerJump <= BboxContinuityEvidence.LOOSE_CENTER_MAX + && areaRatio >= BboxContinuityEvidence.LOOSE_AREA_MIN + && areaRatio <= BboxContinuityEvidence.LOOSE_AREA_MAX; } public int lockClosest(RectF reference) { @@ -115,11 +218,34 @@ public int lockClosest(RectF reference) { bestScore = score; } } - lockedTrackId = best == null ? -1 : best.trackId; - suspectedTrackId = -1; + if (best == null) { + clearLockedTrack("lock_missing"); + return lockedTrackId; + } + lockTrack(best.trackId, "lock"); return lockedTrackId; } + public boolean lockTrack(int trackId, String reason) { + TargetTrack track = getTrackById(trackId); + if (track == null || !track.isVisible()) return false; + lockedTrackId = trackId; + suspectedTrackId = -1; + suspectedDwellFrames = 0; + suspectedBelief = 0f; + suspectedUpdateReason = reason == null ? "lock_track" : reason; + updateLockedGhost(track, 0, track.lastSeenTimestampMs); + return true; + } + + private void clearLockedTrack(String reason) { + lockedTrackId = -1; + suspectedTrackId = -1; + suspectedDwellFrames = 0; + suspectedBelief = 0f; + suspectedUpdateReason = reason == null ? "clear_lock" : reason; + } + public boolean isLockedTrack(TargetTrack track) { return track != null && track.trackId == lockedTrackId; } @@ -150,6 +276,28 @@ private static float area(RectF b) { return b == null ? 0f : Math.max(0f, b.width()) * Math.max(0f, b.height()); } + private void updateLockedGhost(TargetTrack track, int frameW, long timestampMs) { + if (track == null || track.lastBbox == null) return; + if (lockedGhostBbox != null) { + lockedGhostVelocityX = track.lastBbox.centerX() - lockedGhostCenterX; + lockedGhostVelocityY = track.lastBbox.centerY() - lockedGhostCenterY; + } + lockedGhostBbox = new RectF(track.lastBbox); + lockedGhostCenterX = track.lastBbox.centerX(); + lockedGhostCenterY = track.lastBbox.centerY(); + lockedGhostLastSeenMs = timestampMs; + if (frameW > 0) { + lockedGhostLostSide = lockedGhostCenterX < frameW / 2f ? "left" : "right"; + } + } + + private RectF predictedLockedGhost(long nowMs) { + RectF ghost = new RectF(lockedGhostBbox); + float steps = Math.min(3f, Math.max(0f, (nowMs - lockedGhostLastSeenMs) / 250f)); + ghost.offset(lockedGhostVelocityX * steps, lockedGhostVelocityY * steps); + return ghost; + } + private static float iou(RectF a, RectF b) { if (a == null || b == null) return 0f; float left = Math.max(a.left, b.left); diff --git a/android/robot/src/main/java/org/openbot/cartfollow/diagnostics/CartFollowDiagnosticSaver.java b/android/robot/src/main/java/org/openbot/cartfollow/diagnostics/CartFollowDiagnosticSaver.java index 4d26a21d6..f48a81856 100644 --- a/android/robot/src/main/java/org/openbot/cartfollow/diagnostics/CartFollowDiagnosticSaver.java +++ b/android/robot/src/main/java/org/openbot/cartfollow/diagnostics/CartFollowDiagnosticSaver.java @@ -177,7 +177,7 @@ private void appendIdentityLog( String row = String.format( Locale.US, - "%s,%d,%d,%d,%d,%d,%d,%d,%d,%.4f,%.4f,%.4f,%d,%s,%s,%s,%s,%s,%s,%.4f,%d,%d,%d,%s,%s,%s,%s,%s\n", + "%s,%d,%d,%d,%d,%d,%d,%d,%d,%.4f,%.4f,%.4f,%d,%s,%s,%s,%s,%s,%s,%s,%.4f,%d,%d,%d,%s,%s,%s,%s,%s\n", csv(session.sessionId), frameNum, timestampMs, @@ -194,6 +194,7 @@ private void appendIdentityLog( id.weakOk ? "1" : "0", id.midOk ? "1" : "0", id.strongOk ? "1" : "0", + id.bboxLooseAdmissionOk ? "1" : "0", id.bboxDefaultOk ? "1" : "0", id.bboxStrictOk ? "1" : "0", id.predictionOk ? "1" : "0", @@ -314,6 +315,7 @@ private static class IdentitySnapshot { final boolean weakOk; final boolean midOk; final boolean strongOk; + final boolean bboxLooseAdmissionOk; final boolean bboxDefaultOk; final boolean bboxStrictOk; final boolean predictionOk; @@ -340,6 +342,7 @@ private static class IdentitySnapshot { weakOk = identity != null && identity.weakOk(); midOk = identity != null && identity.midOk(); strongOk = identity != null && identity.strongOk(); + bboxLooseAdmissionOk = bbox != null && bbox.looseAdmissionOk; bboxDefaultOk = bbox != null && bbox.bboxDefaultOk; bboxStrictOk = bbox != null && bbox.bboxStrictOk; predictionOk = bbox != null && bbox.predictionOk; diff --git a/android/robot/src/main/java/org/openbot/cartfollow/diagnostics/CartFollowDiagnosticSession.java b/android/robot/src/main/java/org/openbot/cartfollow/diagnostics/CartFollowDiagnosticSession.java index f0e856f9b..f0eef8dcc 100644 --- a/android/robot/src/main/java/org/openbot/cartfollow/diagnostics/CartFollowDiagnosticSession.java +++ b/android/robot/src/main/java/org/openbot/cartfollow/diagnostics/CartFollowDiagnosticSession.java @@ -22,7 +22,7 @@ public class CartFollowDiagnosticSession { private static final String IDENTITY_LOG_HEADER = "session_id,frame_id,timestamp_ms,track_id,locked_track_id,suspected_track_id," + "active_track_count,track_age,missed_frames,best_score,second_score,margin," - + "gallery_size,weak_ok,mid_ok,strong_ok,bbox_default_ok,bbox_strict_ok," + + "gallery_size,weak_ok,mid_ok,strong_ok,bbox_loose_admission_ok,bbox_default_ok,bbox_strict_ok," + "prediction_ok,target_belief,belief_stable_frames,belief_uncertain_frames," + "candidate_switch_count,belief_reason,reid_reason,locked_crop_path," + "suspected_crop_path,best_reid_crop_path"; diff --git a/android/robot/src/main/res/layout/fragment_human_cart_simulator.xml b/android/robot/src/main/res/layout/fragment_human_cart_simulator.xml index 3d4c16ecb..ed3d244a5 100644 --- a/android/robot/src/main/res/layout/fragment_human_cart_simulator.xml +++ b/android/robot/src/main/res/layout/fragment_human_cart_simulator.xml @@ -241,6 +241,15 @@ android:layout_height="match_parent" android:layout_weight="0.2" /> + +