diff --git a/android/cartfollow-devlog.md b/android/cartfollow-devlog.md index 71340bac0..d842e20ff 100644 --- a/android/cartfollow-devlog.md +++ b/android/cartfollow-devlog.md @@ -762,3 +762,124 @@ $env:Path="$env:JAVA_HOME\bin;$env:Path" 5. 默认左上角不再显示大块完整 debug；点击 `调试详情` 后可展开完整字段。 6. Stop / Cancel / Retake / 页面暂停后，诊断 session 应关闭，按钮禁用并复位。 7. 导出诊断目录后，应能用 `events.csv` 附近的 `frame_log.csv` 和 `identity_log.csv` 判断黄框不转绿或非目标转绿的原因。 + +--- + +## 13. Phase C ReID 输入方向修正（2026-07-08） + +### 13.1 修正原因 + +PC 侧 `cartfollow_diagnostics` 复盘发现，旧版诊断 gallery 全部被标记为 `landscape_or_rotated`。代码检查确认旧版 `ReIDCoordinator` 直接在原始 `workingFrame` 上按 bbox 裁剪，并立即送入 `TfliteReIDFeatureExtractor`，没有按 `sensorOrientation` 将 person crop 转正。 + +这意味着旧版 Android ReID 虽然可以运行并输出分数，但 gallery 与候选 crop 都可能以横向姿态进入模型，影响目标返回和多人干扰场景下的稳定性。 + +### 13.2 本轮代码变化 + +- `ReIDCoordinator.collectInitializationCandidate()` 增加 `sensorOrientation` 参数，gallery candidate crop 裁剪后旋转为 upright 再提取 embedding。 +- `ReIDCoordinator.evaluate()` 和内部 ReID candidate 推理同样使用 upright crop。 +- `HumanCartSimulatorFragment` 调用 ReIDCoordinator 时传入当前 `sensorOrientation`。 +- 诊断 `gallery/` 中保存的初始化候选 crop 改为 upright 版本。 +- `session_info.json` 写入 `reid_crop_upright=true` 和 `sensor_orientation`。 +- debug 简洁面板和完整面板增加 `reidCrop=upright`，用于实机确认新版路径已生效。 + +本轮没有修改 ReID 阈值、belief 阈值、bbox gate、状态机恢复规则或真实底盘控制路径。 + +### 13.3 构建验证 + +构建命令： + +```powershell +$env:JAVA_HOME='D:\Java\jdk-17' +$env:Path="$env:JAVA_HOME\bin;$env:Path" +.\gradlew.bat :robot:assembleDebug +``` + +结果：构建通过。构建日志仍有既有 TensorFlow Lite manifest namespace warning、Kotlin/Javac target warning 和 deprecated Gradle warning，均未阻塞构建。 + +### 13.4 下一轮手机验收重点 + +1. Human Cart Simulator debug 应显示 `reidCrop=upright`。 +2. 新采集的 `session_info.json` 应包含 `reid_crop_upright=true`。 +3. 新采集的 `gallery/` 不应再全部被 PC 分析脚本标记为 `landscape_or_rotated`。 +4. 对比旧数据，观察 `target_return` 后 `frames_to_reacquire / frames_to_follow` 是否缩短。 +5. 若 upright crop 后仍大量出现 `belief_high_bbox_failed`，下一轮再处理分状态 bbox gate 和 recoverable stop。 + +--- + +## 14. Phase C 新旧诊断数据对比结论（2026-07-08） + +### 14.1 分析输入 + +本轮没有继续改 Android 策略，而是先对新旧 `cartfollow_diagnostics` 做 PC 离线对比： + +```text +old: tools/reid_pc_test/images/cartfollow_diagnostics_old/ +new: tools/reid_pc_test/images/cartfollow_diagnostics/ +``` + +新版数据来自 ReID crop upright 修正后的 APK，`session_info.json` 中应出现： + +```text +reid_crop_upright=true +sensor_orientation=90 +``` + +PC 分析命令： + +```powershell +cd tools/reid_pc_test +python analyze_cartfollow_diagnostics_v1.py ^ + --compare-roots old=images/cartfollow_diagnostics_old,new=images/cartfollow_diagnostics ^ + --output outputs/cartfollow_diagnostics_analysis/compare +``` + +### 14.2 对比结果 + +| 数据 | sessions | target_return | recovered_rate | recovered_fast | recovered_slow | not_recovered | hard_stop | best_mean | margin_mean | bbox_default_rate | gallery_candidate_landscape_rate | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| old | 4 | 11 | 0.5455 | 5 | 1 | 3 | 2 | 0.5017 | 0.3431 | 0.4451 | 1.0000 | +| new | 2 | 16 | 0.8750 | 11 | 3 | 2 | 0 | 0.5992 | 0.4611 | 0.5485 | 0.0000 | + +结论： + +- upright crop 修正确实生效：新版 `gallery_candidate_landscape_rate=0.0000`。 +- ReID 分数整体提高：`best_mean` 和 `margin_mean` 都高于旧版。 +- `target_return` 后恢复率提升，且新版暂未出现 `hard_stop_before_return`。 +- 这轮结果说明 ReID 输入方向已不是主要问题。 + +### 14.3 当前剩余问题 + +新版主要 blocker： + +```text +candidate_switch_penalty: 15 +belief_high_bbox_failed: 10 +``` + +含义： + +- `candidate_switch_penalty` 说明目标返回或多人干扰时，suspected track 仍容易切换，trackId / lockedTrackId 保护还不够稳。 +- `belief_high_bbox_failed` 说明 ReID / belief 已经有较强证据，但 bbox default/strict gate 不通过，导致黄框迟迟不能转绿。 + +因此下一轮 Android 工作不应继续优先调 ReID 模型、TFLite 性能或 `bestScore / margin` 阈值，而应聚焦： + +1. `TargetTrackManager` 的 track association 稳定性。 +2. locked track 的保留与 suspected track 升级规则。 +3. `FOLLOW` 与 `REACQUIRE/IDENTITY_UNCERTAIN/SEARCH` 使用不同 bbox gate。 +4. recoverable stop 与 hard `STOP` 的边界。 + +### 14.4 下一轮验收口径 + +下一轮策略改动后继续采集 `cartfollow_diagnostics`，重点比较： + +```text +recovered_rate +mean_ms_to_follow +not_recovered_in_window +candidate_switch_penalty +belief_high_bbox_failed +非目标转绿是否增加 +gallery_candidate_landscape_rate 是否保持 0 +``` + +如果 `candidate_switch_penalty` 和 `belief_high_bbox_failed` 下降，同时非目标转绿不增加，才说明策略改动真正改善了“目标返回后迟迟不转绿”和“干扰者偶发转绿”两个问题。 diff --git a/android/robot/src/main/java/org/openbot/cartfollow/HumanCartSimulatorFragment.java b/android/robot/src/main/java/org/openbot/cartfollow/HumanCartSimulatorFragment.java index 20a727c60..e8fbbba98 100644 --- a/android/robot/src/main/java/org/openbot/cartfollow/HumanCartSimulatorFragment.java +++ b/android/robot/src/main/java/org/openbot/cartfollow/HumanCartSimulatorFragment.java @@ -372,9 +372,10 @@ protected void processFrame(Bitmap bitmap, ImageProxy image) { Detector.Recognition largestPerson = selectLargest(mappedRecognitions); if (currentState == FollowState.CAPTURE_TARGET) { if (reidCoordinator != null) { - reidCoordinator.collectInitializationCandidate(workingFrame, largestPerson); + reidCoordinator.collectInitializationCandidate( + workingFrame, largestPerson, sensorOrientation); } - maybeSaveGalleryCandidate(workingFrame, largestPerson); + maybeSaveGalleryCandidate(workingFrame, largestPerson, sensorOrientation); } TargetMatcher.MatchResult legacyMatch = matcher.match( @@ -389,6 +390,7 @@ protected void processFrame(Bitmap bitmap, ImageProxy image) { currentState, frameW, frameH, + sensorOrientation, legacyMatch.score, legacyMatch.matched, legacyMatch.best); @@ -616,7 +618,13 @@ private void activateDiagnosticSession() { boolean reidAvailable = reidCoordinator != null && reidCoordinator.isAvailable(); int gallerySize = reidCoordinator == null ? 0 : reidCoordinator.getGallerySize(); diagnosticSession.writeSessionInfo( - diagnosticConfig, detectorName, minConfidence, reidAvailable, gallerySize); + diagnosticConfig, + detectorName, + minConfidence, + reidAvailable, + gallerySize, + true, + sensorOrientation); resetTargetEventButton(); Toast.makeText( requireContext(), @@ -703,7 +711,8 @@ private void maybeSaveDiagnostics( shouldSaveCrop); } - private void maybeSaveGalleryCandidate(Bitmap frame, Detector.Recognition candidate) { + private void maybeSaveGalleryCandidate( + Bitmap frame, Detector.Recognition candidate, int sensorOrientation) { if (diagnosticSession == null || frame == null || candidate == null) return; if (candidate.getLocation() == null) return; long now = SystemClock.elapsedRealtime(); @@ -711,7 +720,8 @@ private void maybeSaveGalleryCandidate(Bitmap frame, Detector.Recognition candid && now - lastDiagnosticGalleryMs < diagnosticConfig.cropIntervalMs) { return; } - Bitmap crop = cropPerson(frame, candidate.getLocation(), diagnosticConfig.paddingRatio); + Bitmap crop = + cropPerson(frame, candidate.getLocation(), diagnosticConfig.paddingRatio, sensorOrientation); if (crop == null) return; lastDiagnosticGalleryMs = now; diagnosticSaver.saveGallerySnapshotAsync( @@ -723,7 +733,8 @@ private static Detector.Recognition recognitionForTrack(TargetTrack track) { return track == null || !track.isVisible() ? null : track.recognition; } - private static Bitmap cropPerson(Bitmap frame, RectF bbox, float paddingRatio) { + private static Bitmap cropPerson( + Bitmap frame, RectF bbox, float paddingRatio, int sensorOrientation) { if (frame == null || bbox == null) return null; float padX = bbox.width() * paddingRatio; float padY = bbox.height() * paddingRatio; @@ -735,7 +746,15 @@ private static Bitmap cropPerson(Bitmap frame, RectF bbox, float paddingRatio) { int height = bottom - top; if (width <= 0 || height <= 0) return null; try { - return Bitmap.createBitmap(frame, left, top, width, height); + Bitmap rawCrop = Bitmap.createBitmap(frame, left, top, width, height); + int rotation = ((sensorOrientation % 360) + 360) % 360; + if (rotation == 0) return rawCrop; + Matrix matrix = new Matrix(); + matrix.postRotate(rotation); + Bitmap upright = + Bitmap.createBitmap(rawCrop, 0, 0, rawCrop.getWidth(), rawCrop.getHeight(), matrix, true); + rawCrop.recycle(); + return upright; } catch (Exception e) { return null; } @@ -813,7 +832,7 @@ private void updateDebugInfo( String compactInfo = String.format( Locale.US, - "fps=%.1f\nstate=%s\naction=%s\npersons=%d\ntrack=%d locked=%d suspected=%d\nbelief=%.2f\nbest=%.3f margin=%.3f", + "fps=%.1f\nstate=%s\naction=%s\npersons=%d\ntrack=%d locked=%d suspected=%d\nbelief=%.2f\nbest=%.3f margin=%.3f\nreidCrop=upright", fps, state.name(), behaviorDecision == null ? "-" : behaviorDecision.selectedAction.name(), @@ -834,7 +853,7 @@ private void updateDebugInfo( private String buildIdentityDebugLine(IdentityEvidence identity) { if (identity == null) { - return "reidAvailable=false\ngallerySize=0\nbestScore=0.000\nsecondScore=0.000\nmargin=0.000\nweak/mid/strong=false/false/false\nbboxDefault=false bboxStrict=false prediction=false\nstableMatchCount=0\ncandidateSwitchCount=0\nreidLatencyMs=0\nreidReason=-\nactiveTrackCount=0\ntrackId=-1 lockedTrackId=-1 suspectedTrackId=-1\ntrackAge=0 missedFrames=0\nbelief=0.00 beliefStable=0 beliefUncertain=0\nbeliefReason=-"; + return "reidAvailable=false\nreidCrop=upright\ngallerySize=0\nbestScore=0.000\nsecondScore=0.000\nmargin=0.000\nweak/mid/strong=false/false/false\nbboxDefault=false bboxStrict=false prediction=false\nstableMatchCount=0\ncandidateSwitchCount=0\nreidLatencyMs=0\nreidReason=-\nactiveTrackCount=0\ntrackId=-1 lockedTrackId=-1 suspectedTrackId=-1\ntrackAge=0 missedFrames=0\nbelief=0.00 beliefStable=0 beliefUncertain=0\nbeliefReason=-"; } ReIDMatchResult reid = identity.reidMatch; BboxContinuityEvidence bbox = identity.bboxContinuity; @@ -847,7 +866,7 @@ private String buildIdentityDebugLine(IdentityEvidence identity) { String reason = reid == null ? identity.reason : reid.reason; return String.format( Locale.US, - "reidAvailable=%s\ngallerySize=%d\nbestScore=%.3f\nsecondScore=%.3f\nmargin=%.3f\nweak/mid/strong=%s/%s/%s\nbboxDefault=%s bboxStrict=%s prediction=%s\nstableMatchCount=%d\ncandidateSwitchCount=%d\nreidLatencyMs=%d\nreidReason=%s\nactiveTrackCount=%d\ntrackId=%d lockedTrackId=%d suspectedTrackId=%d\ntrackAge=%d missedFrames=%d\nbelief=%.2f reidC=%.2f bboxC=%.2f predC=%.2f switchP=%.2f\nbeliefStable=%d beliefUncertain=%d\nbeliefReason=%s", + "reidAvailable=%s\nreidCrop=upright\ngallerySize=%d\nbestScore=%.3f\nsecondScore=%.3f\nmargin=%.3f\nweak/mid/strong=%s/%s/%s\nbboxDefault=%s bboxStrict=%s prediction=%s\nstableMatchCount=%d\ncandidateSwitchCount=%d\nreidLatencyMs=%d\nreidReason=%s\nactiveTrackCount=%d\ntrackId=%d lockedTrackId=%d suspectedTrackId=%d\ntrackAge=%d missedFrames=%d\nbelief=%.2f reidC=%.2f bboxC=%.2f predC=%.2f switchP=%.2f\nbeliefStable=%d beliefUncertain=%d\nbeliefReason=%s", reidAvailable, gallerySize, best, diff --git a/android/robot/src/main/java/org/openbot/cartfollow/ReIDCoordinator.java b/android/robot/src/main/java/org/openbot/cartfollow/ReIDCoordinator.java index 5614f64d3..75cbd38e3 100644 --- a/android/robot/src/main/java/org/openbot/cartfollow/ReIDCoordinator.java +++ b/android/robot/src/main/java/org/openbot/cartfollow/ReIDCoordinator.java @@ -2,6 +2,7 @@ import android.app.Activity; import android.graphics.Bitmap; +import android.graphics.Matrix; import android.graphics.RectF; import android.os.SystemClock; import java.io.IOException; @@ -77,13 +78,15 @@ public ReIDMatchResult getLastResult() { return lastResult; } - public void collectInitializationCandidate(Bitmap frame, Recognition candidate) { + public void collectInitializationCandidate( + Bitmap frame, Recognition candidate, int sensorOrientation) { if (!isAvailable() || frame == null || candidate == null || candidate.getLocation() == null) { return; } if (pendingGallery.size() >= MAX_PENDING_GALLERY) return; - Bitmap crop = cropPerson(frame, candidate.getLocation(), 0.08f); + Bitmap crop = cropPerson(frame, candidate.getLocation(), 0.08f, sensorOrientation); float[] feature = extractor.extract(crop); + crop.recycle(); if (feature != null) pendingGallery.add(feature); } @@ -105,11 +108,12 @@ public IdentityEvidence evaluate( FollowState state, int frameW, int frameH, + int sensorOrientation, float legacyScore, boolean legacyMatched, Recognition legacyBest) { ReIDMatchResult result = - maybeRunReID(persons, frame, memory, state, frameW, frameH, legacyBest); + maybeRunReID(persons, frame, memory, state, frameW, frameH, sensorOrientation, legacyBest); Recognition best = lastBestCandidate != null ? lastBestCandidate : legacyBest; boolean matched = legacyMatched; float confidence = legacyScore; @@ -136,6 +140,7 @@ private ReIDMatchResult maybeRunReID( FollowState state, int frameW, int frameH, + int sensorOrientation, Recognition legacyBest) { if (!isAvailable()) { lastResult = ReIDMatchResult.unavailable(disabledReason, 0); @@ -166,8 +171,9 @@ private ReIDMatchResult maybeRunReID( long start = SystemClock.elapsedRealtime(); List scores = new ArrayList<>(); for (CandidateRef ref : candidateRefs(persons, legacyBest)) { - Bitmap crop = cropPerson(frame, ref.recognition.getLocation(), 0.08f); + Bitmap crop = cropPerson(frame, ref.recognition.getLocation(), 0.08f, sensorOrientation); float[] feature = extractor.extract(crop); + crop.recycle(); if (feature == null) continue; float score = maxSimilarity(feature, confirmedGallery); scores.add(new CandidateScore(ref.index, ref.recognition, score)); @@ -241,7 +247,8 @@ private static List candidateRefs(List persons, Recog return refs; } - private static Bitmap cropPerson(Bitmap frame, RectF bbox, float paddingRatio) { + private static Bitmap cropPerson( + Bitmap frame, RectF bbox, float paddingRatio, int sensorOrientation) { int fw = frame.getWidth(); int fh = frame.getHeight(); float padX = bbox.width() * paddingRatio; @@ -252,7 +259,15 @@ private static Bitmap cropPerson(Bitmap frame, RectF bbox, float paddingRatio) { int bottom = clamp((int) (bbox.bottom + padY), 1, fh); int width = Math.max(1, right - left); int height = Math.max(1, bottom - top); - return Bitmap.createBitmap(frame, left, top, width, height); + Bitmap rawCrop = Bitmap.createBitmap(frame, left, top, width, height); + int rotation = ((sensorOrientation % 360) + 360) % 360; + if (rotation == 0) return rawCrop; + Matrix matrix = new Matrix(); + matrix.postRotate(rotation); + Bitmap upright = + Bitmap.createBitmap(rawCrop, 0, 0, rawCrop.getWidth(), rawCrop.getHeight(), matrix, true); + rawCrop.recycle(); + return upright; } private static List selectDiverse(List features, int k) { diff --git a/android/robot/src/main/java/org/openbot/cartfollow/diagnostics/CartFollowDiagnosticSession.java b/android/robot/src/main/java/org/openbot/cartfollow/diagnostics/CartFollowDiagnosticSession.java index eb0ee3b89..f0e856f9b 100644 --- a/android/robot/src/main/java/org/openbot/cartfollow/diagnostics/CartFollowDiagnosticSession.java +++ b/android/robot/src/main/java/org/openbot/cartfollow/diagnostics/CartFollowDiagnosticSession.java @@ -75,7 +75,9 @@ public void writeSessionInfo( String detectorModelName, float minConfidence, boolean reidAvailable, - int gallerySize) { + int gallerySize, + boolean reidCropUpright, + int sensorOrientation) { JSONObject json = new JSONObject(); try { json.put("session_id", sessionId); @@ -94,6 +96,8 @@ public void writeSessionInfo( json.put("min_confidence", minConfidence); json.put("reid_available", reidAvailable); json.put("gallery_size", gallerySize); + json.put("reid_crop_upright", reidCropUpright); + json.put("sensor_orientation", sensorOrientation); json.put("device_model", Build.MODEL == null ? "" : Build.MODEL); json.put("sdk_int", Build.VERSION.SDK_INT); } catch (JSONException e) {