diff --git a/android/cartfollow-devlog.md b/android/cartfollow-devlog.md index 0aef4c1d1..5549681e4 100644 --- a/android/cartfollow-devlog.md +++ b/android/cartfollow-devlog.md @@ -288,10 +288,152 @@ STOP ──(用户重新开始)──→ CAPTURE_TARGET | `6d9aa5f` | 2026-07-06 | Add capture controls and status panel | | `765eb82` | 2026-07-06 | Put Person ID input on its own row for tap accessibility | | `771345e` | 2026-07-06 | Rotate crop by sensorOrientation before saving to disk | +| pending | 2026-07-07 | Add PersonSequenceCollector for continuous sequence data collection | --- -## 7. 调试提示 +## 7. Person Sequence Collector（Phase 3 时序数据采集） + +> 更新日期：2026-07-07 +> 代码位置：`dev/OpenBot/android/robot/src/main/java/org/openbot/sequencecollector/` +> 目的：为 PC 端 chronological replay / 状态机回放采集连续时序事实数据。 +> 当前状态：已实现、已构建通过、已安装到手机，并完成两条真实 sequence 采集。 + +### 7.1 模块定位 + +`PersonSequenceCollector` 是独立于 `PersonCropCollector` 的采集工具。它不做 ReID 推理，不控制小车，不写入 `FOLLOW / LOST / REACQUIRE / STOP` 等状态标签，只记录摄像头检测到的事实： + +```text +每个采样帧是否有人； +每个采样帧有几个人； +每个检测框的 bbox / confidence / crop_path； +人工标记的 target_left / target_return / occlusion / distractor 事件。 +``` + +这样 PC 端可以用同一份时序数据复现目标离开、遮挡、返回、干扰者进入等场景，而不是继续依赖随机抽样 rows。 + +### 7.2 新增文件 + +| 文件 | 作用 | +|------|------| +| `sequencecollector/PersonSequenceCollectorFragment.java` | 独立 CameraFragment 页面，复用 OpenBot Detector 检测 person，写入连续时序日志。 | +| `sequencecollector/PersonSequenceCaptureConfig.java` | 管理 frame log / crop / overlay 采样间隔、置信度、是否保存 crop 等配置。 | +| `sequencecollector/PersonSequenceSession.java` | 创建 `cartfollow_sequences//`，初始化 CSV 和 `session_info.json`。 | +| `sequencecollector/PersonSequenceSaver.java` | 单线程异步写 `frame_log.csv`、`detections.csv`、`events.csv` 和可选 crops。 | +| `res/layout/fragment_person_sequence_collector.xml` | Sequence 采集 UI。 | + +入口集成： + +| 文件 | 修改内容 | +|------|----------| +| `FeatureList.java` | 新增 `PERSON_SEQUENCE_COLLECTOR` 主菜单项。 | +| `MainFragment.java` | 新增跳转到 `personSequenceCollectorFragment`。 | +| `nav_graph.xml` | 注册 `personSequenceCollectorFragment`。 | +| `strings.xml` | 新增 Sequence Collector 标题、Start/Stop、idle 文案。 | + +### 7.3 输出目录与文件 + +输出目录位于 App 外部图片目录： + +```text +/sdcard/Android/data/org.openbot/files/Pictures/cartfollow_sequences/ +└── _seq_/ + ├── frame_log.csv + ├── detections.csv + ├── events.csv + ├── session_info.json + ├── crops/ + └── overlays/ +``` + +CSV 字段： + +```text +frame_log.csv: +session_id,frame_id,timestamp_ms,elapsed_ms,image_width,image_height,num_persons,raw_frame_path,overlay_path,event_tag,note + +detections.csv: +session_id,frame_id,det_id,timestamp_ms,confidence,bbox_left,bbox_top,bbox_right,bbox_bottom,bbox_width,bbox_height,bbox_area_ratio,center_x,center_y,edge_touch,crop_path + +events.csv: +session_id,timestamp_ms,frame_id,event_type,note +``` + +当前默认参数： + +| 参数 | 默认值 | +|------|--------| +| `frameLogIntervalMs` | 200 ms | +| `cropIntervalMs` | 500 ms | +| `overlayIntervalMs` | 1000 ms | +| `minConfidence` | 0.5 | +| `saveCrops` | true | +| `saveOverlays` | false | +| `jpegQuality` | 90 | + +说明：`frameLogIntervalMs` 与 `cropIntervalMs` 可在页面中通过 +/- 控件调整。第二条 sequence `yrc2_seq_20260707_152237` 实测使用 `cropIntervalMs=300 ms`，用于提高 PC 端 ReID replay 的帧密度。 + +### 7.4 当前验证状态 + +已完成静态构建验证： + +```powershell +$env:JAVA_HOME='D:\Java\jdk-17' +.\gradlew.bat :robot:assembleDebug +``` + +结果：构建通过。默认 JDK 24 会触发 Android Gradle `jlink` 兼容问题，需使用本机 `D:\Java\jdk-17` 构建。 + +已完成真机验证： + +```text +主菜单能看到 Person Sequence Collector； +进入页面后能显示 person bbox； +Start 后创建 cartfollow_sequences//； +无人帧写入 frame_log.csv，num_persons=0； +多人帧在 detections.csv 写多行； +事件按钮能追加 events.csv； +Stop 后显示 frames / detections / crops / events 和导出路径； +adb pull 后 PC sequence replay 可以读取并扩展使用。 +``` + +已采集数据： + +| sequence | 说明 | PC 侧结论 | +|----------|------|-----------| +| `yrc_seq_20260707_140056` | 首条真实 sequence，包含目标离开、返回、干扰者、遮挡事件。 | 安全性成立，未出现错误恢复 FOLLOW；高 over-stop 主要来自终态 STOP 后尾段。 | +| `yrc2_seq_20260707_152237` | 更结构化 sequence：正常跟随、目标离开、无人帧、返回、干扰者进入/离开、遮挡。 | 暴露“看到了目标但恢复条件太严”的问题；宽松恢复条件可避免 STOP，并保持 `wrong_recovery_count=0`。 | + +### 7.5 对 FollowStateMachine 的最新启发 + +第二条 sequence 表明：目标返回后，系统经常能看到连续稳定 bbox 和中等偏高 ReID 分数，但如果恢复条件只接受很强的 `strong + strict` 连续证据，就会长期卡在 `IDENTITY_UNCERTAIN`，最后超时进入 `STOP`。 + +后续 Android 状态机不应把 `STOP` 当成唯一安全动作，而应区分： + +```text +motion_stop: + 线速度为 0，不继续前进，但仍观察、原地搜索、尝试重捕获。 + +hard STOP: + 搜索失败、风险过高或安全异常后的终态停车，等待人工重新开始。 +``` + +建议新增或细化状态： + +```text +FOLLOW +FOLLOW_CAUTION +IDENTITY_UNCERTAIN +LOCAL_SEARCH +REACQUIRE_TARGET +STOP +``` + +目标丢失时应先进入 `motion_stop + LOCAL_SEARCH`，根据最后 bbox 方向做原地低速搜索；目标返回后若连续多帧满足 `ReID + bbox + prediction` 稳定证据，再进入 `REACQUIRE_TARGET`，最后恢复 `FOLLOW`。只有搜索超时、干扰风险过高、障碍/急停/通信异常时才进入 hard `STOP`。 + +--- + +## 8. 调试提示 - 使用 `/dev/OpenBot/android` 在 Android Studio 中打开工程 - 主菜单 → "Cart Simulator" 进入本模块 @@ -301,3 +443,5 @@ STOP ──(用户重新开始)──→ CAPTURE_TARGET - 主菜单 → "Person Crop Collector" 进入 ReID 数据采集页 - 输入 Person ID，保持 `Single Only` 打开，点击 Start Session 后采集真实 person crop - 采集目录导出到 PC 后，用 `tools/reid_pc_test/prepare_openbot_crops_dataset.py` 整理为 `images_openbot_clean/` +- 主菜单 → "Person Sequence Collector" 进入连续时序采集页 +- Sequence 采集时事件按钮只需在事件开始/结束时各按一次，PC replay 会用容忍窗口处理人工反应延迟 diff --git a/android/robot/src/main/java/org/openbot/common/FeatureList.java b/android/robot/src/main/java/org/openbot/common/FeatureList.java index a5fab63e3..39639f852 100644 --- a/android/robot/src/main/java/org/openbot/common/FeatureList.java +++ b/android/robot/src/main/java/org/openbot/common/FeatureList.java @@ -38,6 +38,7 @@ public class FeatureList { public static final String OBJECT_NAV = "Object Tracking"; public static final String CART_SIMULATOR = "Cart Simulator"; public static final String PERSON_CROP_COLLECTOR = "Person Crop Collector"; + public static final String PERSON_SEQUENCE_COLLECTOR = "Person Sequence Collector"; public static final String MODEL_MANAGEMENT = "Model Management"; public static final String POINT_GOAL_NAVIGATION = "Point Goal Navigation"; public static final String AUTONOMOUS_DRIVING = "Autonomous Driving"; @@ -87,6 +88,8 @@ public static ArrayList getCategories() { subCategories.add(new SubCategory(CART_SIMULATOR, R.drawable.ic_person_search, "#6BBF8A")); subCategories.add( new SubCategory(PERSON_CROP_COLLECTOR, R.drawable.ic_person_search, "#8BBF6B")); + subCategories.add( + new SubCategory(PERSON_SEQUENCE_COLLECTOR, R.drawable.ic_person_search, "#6BA9BF")); subCategories.add( new SubCategory(POINT_GOAL_NAVIGATION, R.drawable.ic_baseline_golf_course, "#1BBFBF")); subCategories.add(new SubCategory(MODEL_MANAGEMENT, R.drawable.ic_list_bulleted_48, "#BC7680")); diff --git a/android/robot/src/main/java/org/openbot/main/MainFragment.java b/android/robot/src/main/java/org/openbot/main/MainFragment.java index 792bb9709..628e9a1f7 100644 --- a/android/robot/src/main/java/org/openbot/main/MainFragment.java +++ b/android/robot/src/main/java/org/openbot/main/MainFragment.java @@ -97,6 +97,11 @@ public void onItemClick(SubCategory subCategory) { .navigate(R.id.action_mainFragment_to_personCropCollectorFragment); break; + case FeatureList.PERSON_SEQUENCE_COLLECTOR: + Navigation.findNavController(requireView()) + .navigate(R.id.action_mainFragment_to_personSequenceCollectorFragment); + break; + case FeatureList.POINT_GOAL_NAVIGATION: Navigation.findNavController(requireView()) .navigate(R.id.action_mainFragment_to_pointGoalNavigationFragment); diff --git a/android/robot/src/main/java/org/openbot/sequencecollector/PersonSequenceCaptureConfig.java b/android/robot/src/main/java/org/openbot/sequencecollector/PersonSequenceCaptureConfig.java new file mode 100644 index 000000000..e3e291305 --- /dev/null +++ b/android/robot/src/main/java/org/openbot/sequencecollector/PersonSequenceCaptureConfig.java @@ -0,0 +1,13 @@ +package org.openbot.sequencecollector; + +public class PersonSequenceCaptureConfig { + public String personId = ""; + public long frameLogIntervalMs = 200; + public long cropIntervalMs = 500; + public long overlayIntervalMs = 1000; + public float minConfidence = 0.5f; + public boolean saveCrops = true; + public boolean saveOverlays = false; + public float paddingRatio = 0.08f; + public int jpegQuality = 90; +} diff --git a/android/robot/src/main/java/org/openbot/sequencecollector/PersonSequenceCollectorFragment.java b/android/robot/src/main/java/org/openbot/sequencecollector/PersonSequenceCollectorFragment.java new file mode 100644 index 000000000..96e56b3a2 --- /dev/null +++ b/android/robot/src/main/java/org/openbot/sequencecollector/PersonSequenceCollectorFragment.java @@ -0,0 +1,503 @@ +package org.openbot.sequencecollector; + +import android.graphics.Bitmap; +import android.graphics.Canvas; +import android.graphics.Color; +import android.graphics.Matrix; +import android.graphics.Paint; +import android.graphics.RectF; +import android.os.Bundle; +import android.os.Handler; +import android.os.HandlerThread; +import android.os.SystemClock; +import android.view.LayoutInflater; +import android.view.View; +import android.view.ViewGroup; +import android.widget.Toast; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import androidx.camera.core.CameraSelector; +import androidx.camera.core.ImageProxy; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import org.jetbrains.annotations.NotNull; +import org.openbot.R; +import org.openbot.common.CameraFragment; +import org.openbot.databinding.FragmentPersonSequenceCollectorBinding; +import org.openbot.env.ImageUtils; +import org.openbot.tflite.Detector; +import org.openbot.tflite.Model; +import org.openbot.tflite.Network; +import org.openbot.utils.CameraUtils; +import org.openbot.utils.Enums; +import timber.log.Timber; + +public class PersonSequenceCollectorFragment extends CameraFragment { + + private FragmentPersonSequenceCollectorBinding binding; + private Handler handler; + private HandlerThread handlerThread; + + private boolean computingNetwork = false; + private float minConfidence = 0.5f; + + private Detector detector; + private Matrix frameToCropTransform; + private Bitmap croppedBitmap; + private int sensorOrientation; + private Matrix cropToFrameTransform; + + private Model model; + private Network.Device device = Network.Device.CPU; + private int numThreads = -1; + private final String classType = "person"; + + private long lastProcessingTimeMs = -1; + private long frameNum = 0; + + private final PersonSequenceCaptureConfig config = new PersonSequenceCaptureConfig(); + private PersonSequenceSession currentSession = null; + private PersonSequenceSaver saver = null; + private boolean captureEnabled = false; + private long lastFrameLogTimeMs = 0; + private long lastCropTimeMs = 0; + private String pendingEventTag = ""; + private String pendingEventNote = ""; + + private final List drawBoxes = new ArrayList<>(); + private int drawFrameWidth = 0; + private int drawFrameHeight = 0; + private int drawSensorOrientation = 0; + + private final Paint personBoxPaint = new Paint(); + private final Paint boxTextPaint = new Paint(); + + @Override + public void onCreate(@Nullable Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + personBoxPaint.setColor(Color.CYAN); + personBoxPaint.setStyle(Paint.Style.STROKE); + personBoxPaint.setStrokeWidth(6.0f); + boxTextPaint.setColor(Color.WHITE); + boxTextPaint.setTextSize(36.0f); + } + + @Override + public View onCreateView( + @NotNull LayoutInflater inflater, ViewGroup container, Bundle savedInstanceState) { + binding = FragmentPersonSequenceCollectorBinding.inflate(inflater, container, false); + return inflateFragment(binding, inflater, container); + } + + @Override + public void onViewCreated(@NonNull View view, @Nullable Bundle savedInstanceState) { + super.onViewCreated(view, savedInstanceState); + + binding.confidenceValue.setText((int) (minConfidence * 100) + "%"); + binding.plusConfidence.setOnClickListener( + v -> { + int confValue = (int) (minConfidence * 100); + if (confValue >= 95) return; + confValue += 5; + minConfidence = confValue / 100f; + binding.confidenceValue.setText(confValue + "%"); + }); + binding.minusConfidence.setOnClickListener( + v -> { + int confValue = (int) (minConfidence * 100); + if (confValue <= 5) return; + confValue -= 5; + minConfidence = confValue / 100f; + binding.confidenceValue.setText(confValue + "%"); + }); + + List models = getModelNames(f -> f.type.equals(Model.TYPE.DETECTOR)); + initModelSpinner(binding.modelSpinner, models, preferencesManager.getObjectNavModel()); + setAnalyserResolution(Enums.Preview.HD.getValue()); + + binding.trackingOverlay.addCallback(canvas -> drawOverlay(canvas)); + binding.statusText.setText(getString(R.string.person_sequence_idle)); + binding.btnStart.setEnabled(true); + binding.btnStop.setEnabled(false); + + binding.frameIntervalValue.setText(config.frameLogIntervalMs + "ms"); + binding.minusFrameInterval.setOnClickListener(v -> adjustFrameInterval(-100)); + binding.plusFrameInterval.setOnClickListener(v -> adjustFrameInterval(100)); + + binding.cropIntervalValue.setText(config.cropIntervalMs + "ms"); + binding.minusCropInterval.setOnClickListener(v -> adjustCropInterval(-100)); + binding.plusCropInterval.setOnClickListener(v -> adjustCropInterval(100)); + + binding.saveCropsSwitch.setChecked(config.saveCrops); + binding.saveCropsSwitch.setOnCheckedChangeListener((buttonView, isChecked) -> config.saveCrops = isChecked); + + binding.btnStart.setOnClickListener(v -> startCapture()); + binding.btnStop.setOnClickListener(v -> stopCapture()); + binding.btnTargetLeft.setOnClickListener(v -> recordManualEvent("target_left")); + binding.btnTargetReturn.setOnClickListener(v -> recordManualEvent("target_return")); + binding.btnOcclusionStart.setOnClickListener(v -> recordManualEvent("occlusion_start")); + binding.btnOcclusionEnd.setOnClickListener(v -> recordManualEvent("occlusion_end")); + binding.btnDistractorEnter.setOnClickListener(v -> recordManualEvent("distractor_enter")); + binding.btnDistractorLeave.setOnClickListener(v -> recordManualEvent("distractor_leave")); + binding.btnManualNote.setOnClickListener(v -> recordManualEvent("manual_note")); + } + + private void adjustFrameInterval(long deltaMs) { + long next = config.frameLogIntervalMs + deltaMs; + if (next < 100 || next > 2000) return; + config.frameLogIntervalMs = next; + binding.frameIntervalValue.setText(config.frameLogIntervalMs + "ms"); + } + + private void adjustCropInterval(long deltaMs) { + long next = config.cropIntervalMs + deltaMs; + if (next < 200 || next > 3000) return; + config.cropIntervalMs = next; + binding.cropIntervalValue.setText(config.cropIntervalMs + "ms"); + } + + private void startCapture() { + String personId = binding.personIdInput.getText().toString().trim(); + if (personId.isEmpty()) { + Toast.makeText(requireContext(), "Please input Person ID", Toast.LENGTH_SHORT).show(); + return; + } + + config.personId = personId; + config.minConfidence = minConfidence; + + currentSession = new PersonSequenceSession(requireContext(), personId); + currentSession.initCsvFiles(); + currentSession.writeSessionInfo(config, model == null ? "" : model.name); + + if (saver != null) { + saver.shutdown(); + } + saver = new PersonSequenceSaver(); + captureEnabled = true; + lastFrameLogTimeMs = 0; + lastCropTimeMs = 0; + pendingEventTag = ""; + pendingEventNote = ""; + + setCaptureControlsEnabled(false); + binding.statusText.setText("Collecting: " + currentSession.sessionId); + } + + private void stopCapture() { + captureEnabled = false; + if (saver != null) { + saver.shutdown(); + saver = null; + } + String path = currentSession != null ? currentSession.sessionDir.getAbsolutePath() : ""; + int frames = currentSession != null ? currentSession.frameRows : 0; + int detections = currentSession != null ? currentSession.detectionRows : 0; + int crops = currentSession != null ? currentSession.cropCount : 0; + int events = currentSession != null ? currentSession.eventRows : 0; + + setCaptureControlsEnabled(true); + binding.statusText.setText( + String.format( + Locale.US, + "Stopped frames=%d det=%d crops=%d events=%d\n%s", + frames, + detections, + crops, + events, + path)); + } + + private void setCaptureControlsEnabled(boolean enabled) { + binding.btnStart.setEnabled(enabled); + binding.btnStop.setEnabled(!enabled); + binding.personIdInput.setEnabled(enabled); + binding.modelSpinner.setEnabled(enabled); + binding.minusFrameInterval.setEnabled(enabled); + binding.plusFrameInterval.setEnabled(enabled); + binding.minusCropInterval.setEnabled(enabled); + binding.plusCropInterval.setEnabled(enabled); + binding.saveCropsSwitch.setEnabled(enabled); + } + + private void recordManualEvent(String eventType) { + String note = binding.noteInput.getText().toString().trim(); + if (captureEnabled && currentSession != null && saver != null) { + pendingEventTag = eventType; + pendingEventNote = note; + saver.saveEventAsync(currentSession, frameNum, eventType, note); + binding.statusText.setText("Event: " + eventType); + } else { + Toast.makeText(requireContext(), "Start a sequence session first", Toast.LENGTH_SHORT).show(); + } + } + + protected void onInferenceConfigurationChanged() { + computingNetwork = false; + if (croppedBitmap == null) return; + final Network.Device device = getDevice(); + final Model model = getModel(); + final int numThreads = getNumThreads(); + runInBackground(() -> recreateNetwork(model, device, numThreads)); + } + + private void recreateNetwork(Model model, Network.Device device, int numThreads) { + if (model == null) return; + Detector newDetector = null; + try { + newDetector = Detector.create(requireActivity(), model, device, numThreads); + } catch (IllegalArgumentException | IOException e) { + Timber.e(e, "Failed to create network."); + String msg = + model.pathType == Model.PATH_TYPE.URL + ? "Model not downloaded. Download it in Model Management first: " + model.name + : "Failed to load model: " + e.getMessage(); + requireActivity() + .runOnUiThread( + () -> Toast.makeText(requireContext().getApplicationContext(), msg, Toast.LENGTH_LONG).show()); + return; + } + + if (detector != null) { + detector.close(); + } + detector = newDetector; + try { + croppedBitmap = + Bitmap.createBitmap( + detector.getImageSizeX(), detector.getImageSizeY(), Bitmap.Config.ARGB_8888); + frameToCropTransform = + ImageUtils.getTransformationMatrix( + getMaxAnalyseImageSize().getWidth(), + getMaxAnalyseImageSize().getHeight(), + croppedBitmap.getWidth(), + croppedBitmap.getHeight(), + sensorOrientation, + detector.getCropRect(), + detector.getMaintainAspect()); + cropToFrameTransform = new Matrix(); + frameToCropTransform.invert(cropToFrameTransform); + } catch (Exception e) { + Timber.e(e, "Failed to configure detector."); + requireActivity() + .runOnUiThread( + () -> + Toast.makeText( + requireContext().getApplicationContext(), + "Failed to configure model: " + e.getMessage(), + Toast.LENGTH_LONG) + .show()); + } + } + + @Override + public synchronized void onResume() { + croppedBitmap = null; + handlerThread = new HandlerThread("sequence-inference"); + handlerThread.start(); + handler = new Handler(handlerThread.getLooper()); + super.onResume(); + } + + @Override + public synchronized void onPause() { + if (captureEnabled) { + stopCapture(); + } + handlerThread.quitSafely(); + try { + handlerThread.join(); + handlerThread = null; + handler = null; + } catch (final InterruptedException e) { + e.printStackTrace(); + } + super.onPause(); + } + + protected synchronized void runInBackground(final Runnable r) { + if (handler != null) handler.post(r); + } + + @Override + protected void processUSBData(String data) {} + + @Override + protected void processControllerKeyData(String commandType) {} + + @Override + protected void processFrame(Bitmap bitmap, ImageProxy image) { + if (detector == null) { + updateCropImageInfo(); + if (detector == null) return; + } + + ++frameNum; + if (binding == null) return; + if (computingNetwork) return; + + computingNetwork = true; + runInBackground( + () -> { + final Canvas canvas = new Canvas(croppedBitmap); + Bitmap workingFrame = bitmap; + if (lensFacing == CameraSelector.LENS_FACING_FRONT) { + Bitmap flipped = CameraUtils.flipBitmapHorizontal(bitmap); + canvas.drawBitmap(flipped, frameToCropTransform, null); + workingFrame = flipped; + } else { + canvas.drawBitmap(bitmap, frameToCropTransform, null); + } + + if (detector != null) { + final long startTime = SystemClock.elapsedRealtime(); + final List results = + detector.recognizeImage(croppedBitmap, classType); + lastProcessingTimeMs = SystemClock.elapsedRealtime() - startTime; + + final List mappedRecognitions = new ArrayList<>(); + for (final Detector.Recognition result : results) { + final RectF location = result.getLocation(); + if (location != null && result.getConfidence() >= minConfidence) { + cropToFrameTransform.mapRect(location); + result.setLocation(location); + mappedRecognitions.add(result); + } + } + + int frameW = getMaxAnalyseImageSize().getWidth(); + int frameH = getMaxAnalyseImageSize().getHeight(); + updateDrawState(mappedRecognitions, frameW, frameH, sensorOrientation); + float fps = lastProcessingTimeMs > 0 ? 1000f / lastProcessingTimeMs : 0f; + + handleSequenceCapture(workingFrame, mappedRecognitions, frameW, frameH); + updateDebugInfo(mappedRecognitions.size(), fps); + binding.trackingOverlay.postInvalidate(); + } + computingNetwork = false; + }); + } + + private void updateCropImageInfo() { + sensorOrientation = 90 - ImageUtils.getScreenOrientation(requireActivity()); + recreateNetwork(getModel(), getDevice(), getNumThreads()); + } + + private void handleSequenceCapture( + Bitmap workingFrame, List persons, int frameW, int frameH) { + if (!captureEnabled || currentSession == null || saver == null) return; + + long now = System.currentTimeMillis(); + if (now - lastFrameLogTimeMs < config.frameLogIntervalMs) return; + + boolean saveCropsForThisFrame = config.saveCrops && now - lastCropTimeMs >= config.cropIntervalMs; + String eventTag = pendingEventTag; + String eventNote = pendingEventNote; + pendingEventTag = ""; + pendingEventNote = ""; + + saver.saveFrameAsync( + workingFrame, + persons, + currentSession, + config, + frameNum, + frameW, + frameH, + sensorOrientation, + saveCropsForThisFrame, + eventTag, + eventNote); + + lastFrameLogTimeMs = now; + if (saveCropsForThisFrame) { + lastCropTimeMs = now; + } + } + + private synchronized void updateDrawState( + List persons, int frameW, int frameH, int sensorOrientation) { + drawBoxes.clear(); + for (Detector.Recognition r : persons) { + if (r == null || r.getLocation() == null) continue; + drawBoxes.add(new RectF(r.getLocation())); + } + drawFrameWidth = frameW; + drawFrameHeight = frameH; + drawSensorOrientation = sensorOrientation; + } + + private void drawOverlay(Canvas canvas) { + if (drawFrameWidth <= 0 || drawFrameHeight <= 0) return; + final boolean rotated = drawSensorOrientation % 180 == 90; + final float multiplier = + Math.min( + canvas.getHeight() / (float) (rotated ? drawFrameWidth : drawFrameHeight), + canvas.getWidth() / (float) (rotated ? drawFrameHeight : drawFrameWidth)); + Matrix matrix = + ImageUtils.getTransformationMatrix( + drawFrameWidth, + drawFrameHeight, + (int) (multiplier * (rotated ? drawFrameHeight : drawFrameWidth)), + (int) (multiplier * (rotated ? drawFrameWidth : drawFrameHeight)), + drawSensorOrientation, + new RectF(0, 0, 0, 0), + false); + + List snapshot; + synchronized (this) { + snapshot = new ArrayList<>(drawBoxes); + } + int idx = 0; + for (RectF box : snapshot) { + RectF rect = new RectF(box); + matrix.mapRect(rect); + float cornerSize = Math.min(rect.width(), rect.height()) / 8.0f; + canvas.drawRoundRect(rect, cornerSize, cornerSize, personBoxPaint); + canvas.drawText("person " + idx, rect.left, Math.max(0, rect.top - 8), boxTextPaint); + idx++; + } + } + + private void updateDebugInfo(int persons, float fps) { + if (binding == null) return; + int frames = currentSession != null ? currentSession.frameRows : 0; + int detections = currentSession != null ? currentSession.detectionRows : 0; + int crops = currentSession != null ? currentSession.cropCount : 0; + int events = currentSession != null ? currentSession.eventRows : 0; + String info = + String.format( + Locale.US, + "persons=%d\nframes=%d\ndet=%d\ncrops=%d\nevents=%d\nfps=%.1f", + persons, + frames, + detections, + crops, + events, + fps); + requireActivity().runOnUiThread(() -> binding.debugInfo.setText(info)); + } + + protected Model getModel() { + return model; + } + + @Override + protected void setModel(Model model) { + if (this.model != model) { + this.model = model; + preferencesManager.setObjectNavModel(model.name); + onInferenceConfigurationChanged(); + } + } + + protected Network.Device getDevice() { + return device; + } + + protected int getNumThreads() { + return numThreads; + } +} diff --git a/android/robot/src/main/java/org/openbot/sequencecollector/PersonSequenceSaver.java b/android/robot/src/main/java/org/openbot/sequencecollector/PersonSequenceSaver.java new file mode 100644 index 000000000..a27d9066d --- /dev/null +++ b/android/robot/src/main/java/org/openbot/sequencecollector/PersonSequenceSaver.java @@ -0,0 +1,260 @@ +package org.openbot.sequencecollector; + +import android.graphics.Bitmap; +import android.graphics.Matrix; +import android.graphics.RectF; +import java.io.File; +import java.io.FileOutputStream; +import java.io.FileWriter; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import org.openbot.tflite.Detector.Recognition; +import timber.log.Timber; + +public class PersonSequenceSaver { + + private final ExecutorService executor = Executors.newSingleThreadExecutor(); + + public void saveFrameAsync( + Bitmap frame, + List persons, + PersonSequenceSession session, + PersonSequenceCaptureConfig config, + long frameNum, + int imageWidth, + int imageHeight, + int sensorOrientation, + boolean saveCropsForThisFrame, + String eventTag, + String note) { + final long timestampMs = System.currentTimeMillis(); + final long elapsedMs = timestampMs - session.startedAtMs; + final List snapshots = snapshotPersons(persons); + Bitmap.Config bitmapConfig = frame == null || frame.getConfig() == null ? Bitmap.Config.ARGB_8888 : frame.getConfig(); + final Bitmap frameCopy = + (frame != null && saveCropsForThisFrame && config.saveCrops) ? frame.copy(bitmapConfig, false) : null; + final String safeEventTag = eventTag == null ? "" : eventTag; + final String safeNote = note == null ? "" : note; + + executor.execute( + () -> { + appendFrameLog( + session, + frameNum, + timestampMs, + elapsedMs, + imageWidth, + imageHeight, + snapshots.size(), + "", + "", + safeEventTag, + safeNote); + + for (int i = 0; i < snapshots.size(); i++) { + RecognitionSnapshot snapshot = snapshots.get(i); + String cropPath = ""; + if (frameCopy != null) { + cropPath = + saveCrop( + frameCopy, + snapshot, + session, + config, + frameNum, + i, + sensorOrientation); + } + appendDetection(session, frameNum, timestampMs, i, snapshot, imageWidth, imageHeight, cropPath); + } + + if (frameCopy != null) { + frameCopy.recycle(); + } + }); + } + + public void saveEventAsync(PersonSequenceSession session, long frameNum, String eventType, String note) { + final long timestampMs = System.currentTimeMillis(); + final String safeType = eventType == null ? "" : eventType; + final String safeNote = note == null ? "" : note; + executor.execute(() -> appendEvent(session, timestampMs, frameNum, safeType, safeNote)); + } + + private List snapshotPersons(List persons) { + List snapshots = new ArrayList<>(); + if (persons == null) return snapshots; + for (Recognition person : persons) { + if (person == null || person.getLocation() == null) continue; + float confidence = person.getConfidence() == null ? 0f : person.getConfidence(); + snapshots.add(new RecognitionSnapshot(new RectF(person.getLocation()), confidence)); + } + return snapshots; + } + + private void appendFrameLog( + PersonSequenceSession session, + long frameNum, + long timestampMs, + long elapsedMs, + int imageWidth, + int imageHeight, + int numPersons, + String rawFramePath, + String overlayPath, + String eventTag, + String note) { + String row = + String.format( + Locale.US, + "%s,%d,%d,%d,%d,%d,%d,%s,%s,%s,%s\n", + csv(session.sessionId), + frameNum, + timestampMs, + elapsedMs, + imageWidth, + imageHeight, + numPersons, + csv(rawFramePath), + csv(overlayPath), + csv(eventTag), + csv(note)); + append(session.frameLogCsv, row, "frame_log.csv"); + session.frameRows++; + } + + private void appendDetection( + PersonSequenceSession session, + long frameNum, + long timestampMs, + int detId, + RecognitionSnapshot snapshot, + int imageWidth, + int imageHeight, + String cropPath) { + RectF bbox = snapshot.bbox; + float areaRatio = imageWidth > 0 && imageHeight > 0 ? bbox.width() * bbox.height() / (imageWidth * imageHeight) : 0f; + boolean edgeTouch = + bbox.left <= 0 || bbox.top <= 0 || bbox.right >= imageWidth || bbox.bottom >= imageHeight; + String row = + String.format( + Locale.US, + "%s,%d,%d,%d,%.4f,%.1f,%.1f,%.1f,%.1f,%.1f,%.1f,%.6f,%.1f,%.1f,%s,%s\n", + csv(session.sessionId), + frameNum, + detId, + timestampMs, + snapshot.confidence, + bbox.left, + bbox.top, + bbox.right, + bbox.bottom, + bbox.width(), + bbox.height(), + areaRatio, + bbox.centerX(), + bbox.centerY(), + edgeTouch ? "1" : "0", + csv(cropPath)); + append(session.detectionsCsv, row, "detections.csv"); + session.detectionRows++; + } + + private void appendEvent( + PersonSequenceSession session, long timestampMs, long frameNum, String eventType, String note) { + String row = + String.format( + Locale.US, + "%s,%d,%d,%s,%s\n", + csv(session.sessionId), timestampMs, frameNum, csv(eventType), csv(note)); + append(session.eventsCsv, row, "events.csv"); + session.eventRows++; + } + + private String saveCrop( + Bitmap frame, + RecognitionSnapshot snapshot, + PersonSequenceSession session, + PersonSequenceCaptureConfig config, + long frameNum, + int detId, + int sensorOrientation) { + RectF bbox = snapshot.bbox; + float padX = bbox.width() * config.paddingRatio; + float padY = bbox.height() * config.paddingRatio; + int left = clamp((int) (bbox.left - padX), 0, frame.getWidth() - 1); + int top = clamp((int) (bbox.top - padY), 0, frame.getHeight() - 1); + int right = clamp((int) (bbox.right + padX), left + 1, frame.getWidth()); + int bottom = clamp((int) (bbox.bottom + padY), top + 1, frame.getHeight()); + int w = right - left; + int h = bottom - top; + if (w <= 0 || h <= 0) return ""; + + Bitmap rawCrop; + try { + rawCrop = Bitmap.createBitmap(frame, left, top, w, h); + } catch (Exception e) { + return ""; + } + + Bitmap uprightCrop; + if (sensorOrientation % 360 != 0) { + Matrix matrix = new Matrix(); + matrix.postRotate(sensorOrientation); + uprightCrop = Bitmap.createBitmap(rawCrop, 0, 0, rawCrop.getWidth(), rawCrop.getHeight(), matrix, true); + rawCrop.recycle(); + } else { + uprightCrop = rawCrop; + } + + String filename = + String.format(Locale.US, "%06d_person%d_conf%.2f.jpg", frameNum, detId, snapshot.confidence); + File cropFile = new File(session.cropsDir, filename); + try (FileOutputStream fos = new FileOutputStream(cropFile)) { + uprightCrop.compress(Bitmap.CompressFormat.JPEG, config.jpegQuality, fos); + session.cropCount++; + return "crops/" + filename; + } catch (IOException e) { + Timber.e(e, "Failed to save sequence crop"); + return ""; + } finally { + uprightCrop.recycle(); + } + } + + private void append(File file, String row, String label) { + try (FileWriter writer = new FileWriter(file, true)) { + writer.append(row); + } catch (IOException e) { + Timber.e(e, "Failed to append %s", label); + } + } + + public void shutdown() { + executor.shutdown(); + } + + private static int clamp(int v, int lo, int hi) { + return v < lo ? lo : (v > hi ? hi : v); + } + + private static String csv(String value) { + if (value == null) return ""; + String escaped = value.replace("\"", "\"\""); + return "\"" + escaped + "\""; + } + + private static class RecognitionSnapshot { + final RectF bbox; + final float confidence; + + RecognitionSnapshot(RectF bbox, float confidence) { + this.bbox = bbox; + this.confidence = confidence; + } + } +} diff --git a/android/robot/src/main/java/org/openbot/sequencecollector/PersonSequenceSession.java b/android/robot/src/main/java/org/openbot/sequencecollector/PersonSequenceSession.java new file mode 100644 index 000000000..3261855db --- /dev/null +++ b/android/robot/src/main/java/org/openbot/sequencecollector/PersonSequenceSession.java @@ -0,0 +1,107 @@ +package org.openbot.sequencecollector; + +import android.content.Context; +import android.os.Environment; +import android.util.Log; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Locale; +import org.json.JSONException; +import org.json.JSONObject; + +public class PersonSequenceSession { + + private static final String TAG = "PersonSequenceSession"; + + private static final String FRAME_LOG_HEADER = + "session_id,frame_id,timestamp_ms,elapsed_ms,image_width,image_height,num_persons," + + "raw_frame_path,overlay_path,event_tag,note"; + private static final String DETECTIONS_HEADER = + "session_id,frame_id,det_id,timestamp_ms,confidence,bbox_left,bbox_top,bbox_right," + + "bbox_bottom,bbox_width,bbox_height,bbox_area_ratio,center_x,center_y,edge_touch," + + "crop_path"; + private static final String EVENTS_HEADER = "session_id,timestamp_ms,frame_id,event_type,note"; + + public final String sessionId; + public final String personId; + public final File sessionDir; + public final File cropsDir; + public final File overlaysDir; + public final File frameLogCsv; + public final File detectionsCsv; + public final File eventsCsv; + public final long startedAtMs; + + public int frameRows = 0; + public int detectionRows = 0; + public int cropCount = 0; + public int eventRows = 0; + + public PersonSequenceSession(Context context, String personId) { + this.personId = personId; + this.startedAtMs = System.currentTimeMillis(); + String timestamp = new SimpleDateFormat("yyyyMMdd_HHmmss", Locale.US).format(new Date()); + this.sessionId = personId + "_seq_" + timestamp; + + File baseDir = + new File( + context.getExternalFilesDir(Environment.DIRECTORY_PICTURES), + "cartfollow_sequences"); + this.sessionDir = new File(baseDir, sessionId); + this.cropsDir = new File(sessionDir, "crops"); + this.overlaysDir = new File(sessionDir, "overlays"); + this.cropsDir.mkdirs(); + this.overlaysDir.mkdirs(); + this.frameLogCsv = new File(sessionDir, "frame_log.csv"); + this.detectionsCsv = new File(sessionDir, "detections.csv"); + this.eventsCsv = new File(sessionDir, "events.csv"); + } + + public void initCsvFiles() { + writeHeader(frameLogCsv, FRAME_LOG_HEADER); + writeHeader(detectionsCsv, DETECTIONS_HEADER); + writeHeader(eventsCsv, EVENTS_HEADER); + } + + private void writeHeader(File file, String header) { + try (FileWriter writer = new FileWriter(file, false)) { + writer.append(header).append('\n'); + } catch (IOException e) { + Log.e(TAG, "Failed to init csv: " + file.getName(), e); + } + } + + public void writeSessionInfo(PersonSequenceCaptureConfig config, String detectorModelName) { + JSONObject json = new JSONObject(); + try { + json.put("session_id", sessionId); + json.put("person_id", personId); + json.put("created_at", new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US).format(new Date())); + json.put("collector", "PersonSequenceCollector"); + json.put("mode", "SEQUENCE"); + json.put("frame_log_interval_ms", config.frameLogIntervalMs); + json.put("crop_interval_ms", config.cropIntervalMs); + json.put("overlay_interval_ms", config.overlayIntervalMs); + json.put("min_confidence", config.minConfidence); + json.put("save_crops", config.saveCrops); + json.put("save_overlays", config.saveOverlays); + json.put("bbox_padding_ratio", config.paddingRatio); + json.put("jpeg_quality", config.jpegQuality); + json.put("detector", detectorModelName == null ? "" : detectorModelName); + json.put("app_mode", "PersonSequenceCollector"); + } catch (JSONException e) { + Log.e(TAG, "Failed to build session_info.json", e); + return; + } + + File infoFile = new File(sessionDir, "session_info.json"); + try (FileWriter writer = new FileWriter(infoFile, false)) { + writer.write(json.toString(2)); + } catch (IOException | JSONException e) { + Log.e(TAG, "Failed to write session_info.json", e); + } + } +} diff --git a/android/robot/src/main/res/layout/fragment_person_sequence_collector.xml b/android/robot/src/main/res/layout/fragment_person_sequence_collector.xml new file mode 100644 index 000000000..20160bc27 --- /dev/null +++ b/android/robot/src/main/res/layout/fragment_person_sequence_collector.xml @@ -0,0 +1,327 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +