Optimise BLE partial flashing

lib/bluetooth.ts

@@ -44,6 +44,7 @@ import {
 } from "./service-events.js";
 
 import { throwIfUnavailable } from "./availability.js";
+import { truncateHexAfterEof } from "./hex-flash-data-source.js";
 
 type BleClientError = { message: string; errorMessage: string };
 
@@ -510,6 +511,7 @@ class MicrobitWebBluetoothConnectionImpl
 
         const partialFlashResult = await partialFlash(
           connection,
+          boardVersion,
           memoryMap,
           progress,
         );
@@ -663,5 +665,5 @@ const convertDataToMemoryMap = (
   if (data instanceof Uint8Array) {
     return MemoryMap.fromPaddedUint8Array(data);
   }
-  return MemoryMap.fromHex(data);
+  return MemoryMap.fromHex(truncateHexAfterEof(data));
 };

lib/flashing/flashing-makecode.test.ts

@@ -0,0 +1,239 @@
+import { describe, expect, it } from "vitest";
+import { readFileSync } from "fs";
+import { resolve } from "path";
+import MemoryMap from "nrf-intel-hex";
+import {
+  isUniversalHex,
+  separateUniversalHex,
+} from "@microbit/microbit-universal-hex";
+import { findMakeCodeRegionInMemoryMap } from "./flashing-makecode.js";
+import { RegionInfo } from "../partial-flashing-service.js";
+import { truncateHexAfterEof } from "../hex-flash-data-source.js";
+
+const hexDir = resolve(__dirname, "../../examples");
+
+const loadHex = (name: string) => readFileSync(resolve(hexDir, name), "utf-8");
+
+/**
+ * Load a hex file, separate if universal, and return parsed MemoryMaps
+ * keyed by board ID (or 0 for thin hex).
+ */
+const loadParsedHex = (
+  name: string,
+): { boardId: number; memoryMap: MemoryMap }[] => {
+  const raw = loadHex(name);
+  if (isUniversalHex(raw)) {
+    return separateUniversalHex(raw).map((p) => ({
+      boardId: p.boardId,
+      memoryMap: MemoryMap.fromHex(p.hex),
+    }));
+  }
+  return [
+    { boardId: 0, memoryMap: MemoryMap.fromHex(truncateHexAfterEof(raw)) },
+  ];
+};
+
+const fullFlashRegion = (end: number): RegionInfo => ({
+  start: 0,
+  end,
+  hash: "",
+});
+
+// All example hex files with expected regions per board
+const hexFiles = [
+  {
+    // v0 embeds source directly in the flash binary (JSON + LZMA at ~0x315f0)
+    // rather than in custom Intel HEX record type 0x0E like later versions.
+    name: "microbit-small-heart-makecode-v0.hex",
+    label: "MakeCode v0",
+    universal: false,
+    regions: [
+      { boardId: 0, start: 0x30c00, end: 0x3fc40, hash: "AB433FFB2EC06262" },
+    ],
+  },
+  {
+    name: "microbit-small-heart-makecode-v1.hex",
+    label: "MakeCode v1",
+    universal: false,
+    regions: [
+      { boardId: 0, start: 0x32c00, end: 0x3fc40, hash: "13FC432659DAE036" },
+    ],
+  },
+  {
+    name: "microbit-small-heart-makecode-v2.hex",
+    label: "MakeCode v2",
+    universal: false,
+    regions: [
+      { boardId: 0, start: 0x34800, end: 0x3fc40, hash: "02F597B8CE253C03" },
+    ],
+  },
+  {
+    name: "microbit-small-heart-makecode-v3.hex",
+    label: "MakeCode v3",
+    universal: true,
+    regions: [
+      {
+        boardId: 0x9901,
+        start: 0x35000,
+        end: 0x3fc40,
+        hash: "4F4FD28A88324C7E",
+      },
+      {
+        boardId: 0x9903,
+        start: 0x45000,
+        end: 0x7f340,
+        hash: "FFC95E6502F1C747",
+      },
+    ],
+  },
+  {
+    name: "microbit-small-heart-makecode-v4.hex",
+    label: "MakeCode v4",
+    universal: true,
+    regions: [
+      {
+        boardId: 0x9901,
+        start: 0x35000,
+        end: 0x3fc40,
+        hash: "02E34856C566087A",
+      },
+      {
+        boardId: 0x9903,
+        start: 0x47000,
+        end: 0x7f340,
+        hash: "8E4D117FB81B2B19",
+      },
+    ],
+  },
+  {
+    name: "microbit-small-heart-makecode-v5.hex",
+    label: "MakeCode v5",
+    universal: true,
+    regions: [
+      {
+        boardId: 0x9900,
+        start: 0x36800,
+        end: 0x3fc40,
+        hash: "FFDA7A967870CC1B",
+      },
+      {
+        boardId: 0x9903,
+        start: 0x47000,
+        end: 0x7f340,
+        hash: "3BD7683DA9B220E8",
+      },
+    ],
+  },
+  {
+    name: "microbit-small-heart-makecode-v6.hex",
+    label: "MakeCode v6",
+    universal: true,
+    regions: [
+      {
+        boardId: 0x9900,
+        start: 0x35400,
+        end: 0x3fc40,
+        hash: "F4B5733CAAC69708",
+      },
+      {
+        boardId: 0x9903,
+        start: 0x47000,
+        end: 0x7f340,
+        hash: "890F5019E604A158",
+      },
+    ],
+  },
+  {
+    name: "microbit-small-heart-makecode-v7.hex",
+    label: "MakeCode v7",
+    universal: true,
+    regions: [
+      {
+        boardId: 0x9900,
+        start: 0x35400,
+        end: 0x3fc40,
+        hash: "F87DD6F924D49825",
+      },
+      {
+        boardId: 0x9903,
+        start: 0x47000,
+        end: 0x7f340,
+        hash: "42B6FF95F591DA1F",
+      },
+    ],
+  },
+  {
+    name: "microbit-small-heart-makecode-v8.hex",
+    label: "MakeCode v8",
+    universal: true,
+    regions: [
+      {
+        boardId: 0x9900,
+        start: 0x35400,
+        end: 0x3fc40,
+        hash: "8E76592A9CB06B29",
+      },
+      {
+        boardId: 0x9903,
+        start: 0x47000,
+        end: 0x7f340,
+        hash: "22204AC874977E7F",
+      },
+    ],
+  },
+];
+
+describe("findMakeCodeRegionInMemoryMap", () => {
+  for (const { name, label, universal, regions } of hexFiles) {
+    describe(label, () => {
+      const parts = loadParsedHex(name);
+
+      if (!universal) {
+        it("is a non-universal (thin) hex", () => {
+          expect(isUniversalHex(loadHex(name))).toBe(false);
+        });
+      }
+
+      for (const expected of regions) {
+        const part = parts.find((p) => p.boardId === expected.boardId);
+        const isV2Board =
+          expected.boardId === 0x9903 || expected.boardId === 0x9904;
+        const boardLabel =
+          expected.boardId === 0
+            ? "thin V1"
+            : `boardId=0x${expected.boardId.toString(16)}`;
+        const flashEnd = isV2Board ? 0x80000 : 0x40000;
+
+        it(`finds the MakeCode region (${boardLabel})`, () => {
+          expect(part).toBeDefined();
+          const region = findMakeCodeRegionInMemoryMap(
+            part!.memoryMap,
+            fullFlashRegion(flashEnd),
+          );
+          expect(region).toEqual({
+            start: expected.start,
+            end: expected.end,
+            hash: expected.hash,
+          });
+        });
+      }
+    });
+  }
+
+  describe("thin hex EOF truncation", () => {
+    const thinFiles = hexFiles.filter((f) => !f.universal);
+
+    for (const { name, label } of thinFiles) {
+      it(`${label} raw hex has data after EOF that MemoryMap.fromHex rejects`, () => {
+        const raw = loadHex(name);
+        expect(() => MemoryMap.fromHex(raw)).toThrow(/after.*EOF/i);
+      });
+
+      it(`${label} can be parsed after truncating at EOF`, () => {
+        const parts = loadParsedHex(name);
+        expect(parts).toHaveLength(1);
+        expect(parts[0].memoryMap).toBeInstanceOf(MemoryMap);
+      });
+    }
+  });
+});

lib/flashing/flashing-makecode.ts

@@ -26,6 +26,7 @@ export const findMakeCodeRegionInMemoryMap = (
           end = Math.max(end, Math.min(blockEnd, deviceCodeRegion.end));
         }
       }
+
       // Round up to next 64-byte boundary
       end = Math.ceil(end / 64) * 64;
 

lib/flashing/flashing-partial.ts

@@ -7,7 +7,17 @@ import {
 } from "../partial-flashing-service.js";
 import { findMakeCodeRegionInMemoryMap } from "./flashing-makecode.js";
 import { delay, DisconnectError } from "../async-util.js";
-import { DeviceError, ProgressCallback, ProgressStage } from "../device.js";
+import {
+  BoardVersion,
+  DeviceError,
+  ProgressCallback,
+  ProgressStage,
+} from "../device.js";
+
+const FLASH_PAGE_SIZE: Record<BoardVersion, number> = {
+  V1: 0x400,
+  V2: 0x1000,
+};
 
 export enum PartialFlashResult {
   Success = "Success",
@@ -16,14 +26,15 @@ export enum PartialFlashResult {
 
 const partialFlash = async (
   connection: BluetoothDeviceWrapper,
+  boardVersion: BoardVersion,
   memoryMap: MemoryMap,
   progress: ProgressCallback,
 ): Promise<PartialFlashResult> => {
   const pf = new PartialFlashingService(connection);
   await pf.startNotifications();
 
   const result = await connection.raceDisconnectAndTimeout(
-    partialFlashInternal(connection, pf, memoryMap, progress),
+    partialFlashInternal(connection, pf, boardVersion, memoryMap, progress),
     { timeout: 30_000, actionName: "partial flash" },
   );
 
@@ -42,6 +53,7 @@ const partialFlash = async (
 const partialFlashInternal = async (
   connection: BluetoothDeviceWrapper,
   pf: PartialFlashingService,
+  boardVersion: BoardVersion,
   memoryMap: MemoryMap,
   progress: ProgressCallback,
 ): Promise<PartialFlashResult> => {
@@ -82,12 +94,34 @@ const partialFlashInternal = async (
       return PartialFlashResult.AttemptFullFlash;
     }
 
+    // The device-side partial flash service erases each flash page when it
+    // receives a write at a page-aligned address. After erase, every byte
+    // in the page is 0xFF, so writing 0xFF within an already-erased page
+    // is redundant. We skip interior 0xFF blocks but always send at page
+    // boundaries to trigger the erase.
+    //
+    // See the page-erase logic:
+    //   V1: https://github.com/lancaster-university/microbit-dal/blob/master/source/bluetooth/MicroBitPartialFlashingService.cpp
+    //   V2: https://github.com/lancaster-university/codal-microbit-v2/blob/master/source/bluetooth/MicroBitPartialFlashingService.cpp
+    const flashPageSize = FLASH_PAGE_SIZE[boardVersion];
+
     let nextPacketNumber = 0;
     outer: for (
       let offset = fileCodeRegion.start;
       offset < fileCodeRegion.end;
 
     ) {
+      // Skip 64-byte blocks that are entirely 0xFF, unless at a page boundary.
+      // At page boundaries we must always send data so the device erases the
+      // page (setting all bytes to 0xFF). Interior 0xFF blocks can be skipped
+      // because the page has already been erased by the boundary write.
+      if (offset % flashPageSize !== 0) {
+        if (memoryMap.slicePad(offset, 64).every((b) => b === 0xff)) {
+          offset += 64;
+          continue;
+        }
+      }
+
       const batchStartAddress = offset;
 
       for (let packetInBatch = 0; packetInBatch < 4; ++packetInBatch) {

lib/hex-flash-data-source.ts

@@ -9,6 +9,16 @@ import {
   separateUniversalHex,
 } from "@microbit/microbit-universal-hex";
 
+// MakeCode hex files may contain embedded source in custom record types
+// (type 0x0E) after the EOF record, and older files may have trailing blank
+// lines. The nrf-intel-hex parser rejects any data after EOF, so we truncate.
+export const truncateHexAfterEof = (hex: string): string => {
+  // The EOF record is :00000001FF (case-insensitive per the Intel HEX spec).
+  const eofIdx = hex.search(/:00000001FF/i);
+  if (eofIdx < 0) return hex;
+  return hex.slice(0, eofIdx + ":00000001FF".length) + "\n";
+};
+
 /**
  * A flash data source that converts universal hex files as needed.
  *

lib/usb-partial-flashing.ts

@@ -59,6 +59,7 @@ import {
   read32FromUInt8Array,
 } from "./usb-partial-flashing-utils.js";
 import { BoardVersion, ProgressCallback, ProgressStage } from "./device.js";
+import { truncateHexAfterEof } from "./hex-flash-data-source.js";
 import MemoryMap from "nrf-intel-hex";
 
 // Source code for binaries in can be found at https://github.com/microsoft/pxt-microbit/blob/dec5b8ce72d5c2b4b0b20aafefce7474a6f0c7b2/external/sha/source/main.c
@@ -346,7 +347,7 @@ export class PartialFlashing {
   }
 
   private hexStringToPaddedBytes(hex: string): Uint8Array {
-    const m = MemoryMap.fromHex(hex);
+    const m = MemoryMap.fromHex(truncateHexAfterEof(hex));
     return this.memoryMapToPaddedBytes(m);
   }