fix(api): reconcile emhttp state without spinning disks

api/src/__test__/store/modules/emhttp-replacement.test.ts

@@ -0,0 +1,71 @@
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';
+
+const VAR_FIXTURE = readFileSync(new URL('../../../../dev/states/var.ini', import.meta.url), 'utf-8');
+
+const writeVarFixture = (dir: string, { fsState, mdState }: { fsState: string; mdState: string }) => {
+    const content = VAR_FIXTURE.replace(/mdState="[^"]*"/, `mdState="${mdState}"`).replace(
+        /fsState="[^"]*"/,
+        `fsState="${fsState}"`
+    );
+    writeFileSync(join(dir, 'var.ini'), content);
+};
+
+describe('emhttp state reload after file replacement', () => {
+    let tempDir: string;
+
+    beforeEach(() => {
+        vi.resetModules();
+        tempDir = mkdtempSync(join(tmpdir(), 'emhttp-replacement-'));
+    });
+
+    afterEach(() => {
+        vi.restoreAllMocks();
+        rmSync(tempDir, { recursive: true, force: true });
+    });
+
+    test('reloads a replaced var.ini from the stale STARTED/Stopping state seen on Unraid hosts', async () => {
+        writeVarFixture(tempDir, { mdState: 'STOPPED', fsState: 'Stopped' });
+
+        const { store } = await import('@app/store/index.js');
+        const { getArrayData } = await import('@app/core/modules/array/get-array-data.js');
+        const { ArrayState } = await import('@app/unraid-api/graph/resolvers/array/array.model.js');
+        const { loadSingleStateFile, loadStateFiles, updateEmhttpState } = await import(
+            '@app/store/modules/emhttp.js'
+        );
+        const { StateFileKey } = await import('@app/store/types.js');
+
+        const originalGetState = store.getState.bind(store);
+        vi.spyOn(store, 'getState').mockImplementation(() => ({
+            ...originalGetState(),
+            paths: {
+                ...originalGetState().paths,
+                states: tempDir,
+            },
+        }));
+
+        await store.dispatch(loadStateFiles());
+
+        store.dispatch(
+            updateEmhttpState({
+                field: StateFileKey.var,
+                state: {
+                    mdState: ArrayState.STARTED,
+                    fsState: 'Stopping',
+                },
+            })
+        );
+
+        expect(getArrayData(() => store.getState()).state).toBe(ArrayState.STARTED);
+        expect(store.getState().emhttp.var.fsState).toBe('Stopping');
+
+        await store.dispatch(loadSingleStateFile(StateFileKey.var));
+
+        expect(store.getState().emhttp.var.mdState).toBe(ArrayState.STOPPED);
+        expect(store.getState().emhttp.var.fsState).toBe('Stopped');
+        expect(getArrayData(() => store.getState()).state).toBe(ArrayState.STOPPED);
+    });
+});

api/src/__test__/store/watch/state-watch.test.ts

@@ -3,19 +3,30 @@ import { beforeEach, describe, expect, it, vi } from 'vitest';
 import { StateFileKey } from '@app/store/types.js';
 
 type WatchHandler = (path: string) => Promise<void>;
+type WatchRegistration = {
+    path: string;
+    options: Record<string, unknown>;
+    handlers: Partial<Record<'add' | 'change', WatchHandler>>;
+};
 
-const handlersByPath = new Map<string, Partial<Record<'add' | 'change', WatchHandler>>>();
+const watchRegistrations: WatchRegistration[] = [];
 
-const createWatcher = (path: string) => ({
+const createWatcher = (registration: WatchRegistration) => ({
     on: vi.fn((event: 'add' | 'change', handler: WatchHandler) => {
-        const existingHandlers = handlersByPath.get(path) ?? {};
-        existingHandlers[event] = handler;
-        handlersByPath.set(path, existingHandlers);
-        return createWatcher(path);
+        registration.handlers[event] = handler;
+        return createWatcher(registration);
     }),
 });
 
-const chokidarWatch = vi.fn((path: string) => createWatcher(path));
+const chokidarWatch = vi.fn((path: string, options: Record<string, unknown> = {}) => {
+    const registration: WatchRegistration = {
+        path,
+        options,
+        handlers: {},
+    };
+    watchRegistrations.push(registration);
+    return createWatcher(registration);
+});
 
 vi.mock('chokidar', () => ({
     watch: chokidarWatch,
@@ -56,51 +67,138 @@ describe('StateManager', () => {
     beforeEach(async () => {
         vi.resetModules();
         vi.clearAllMocks();
-        handlersByPath.clear();
+        watchRegistrations.length = 0;
 
         const { StateManager } = await import('@app/store/watch/state-watch.js');
         StateManager.instance = null;
     });
 
-    it('watches devs.ini alongside the other emhttp state files', async () => {
+    it('watches the emhttp state directory and keeps polling scoped to replacement-prone files', async () => {
+        const { StateManager } = await import('@app/store/watch/state-watch.js');
+
+        await StateManager.getInstance().ready;
+
+        expect(chokidarWatch).toHaveBeenCalledTimes(3);
+        expect(chokidarWatch).toHaveBeenNthCalledWith(
+            1,
+            '/usr/local/emhttp/state',
+            expect.objectContaining({
+                ignoreInitial: true,
+                usePolling: false,
+                ignored: expect.any(Function),
+            })
+        );
+        expect(chokidarWatch).toHaveBeenNthCalledWith(
+            2,
+            '/usr/local/emhttp/state/disks.ini',
+            expect.objectContaining({
+                ignoreInitial: true,
+                usePolling: true,
+                interval: 10_000,
+            })
+        );
+        expect(chokidarWatch).toHaveBeenNthCalledWith(
+            3,
+            '/usr/local/emhttp/state/shares.ini',
+            expect.objectContaining({
+                ignoreInitial: true,
+                usePolling: true,
+                interval: 10_000,
+            })
+        );
+    });
+
+    it('reconciles all emhttp state files after watchers are attached', async () => {
         const { StateManager } = await import('@app/store/watch/state-watch.js');
+        const { store } = await import('@app/store/index.js');
+        const { loadSingleStateFile } = await import('@app/store/modules/emhttp.js');
+        const { loadRegistrationKey } = await import('@app/store/modules/registration.js');
 
-        StateManager.getInstance();
+        await StateManager.getInstance().ready;
 
-        expect(chokidarWatch).toHaveBeenCalledWith('/usr/local/emhttp/state/devs.ini', {
-            usePolling: false,
-        });
+        expect(store.dispatch).toHaveBeenNthCalledWith(1, loadSingleStateFile(StateFileKey.var));
+        expect(store.dispatch).toHaveBeenNthCalledWith(2, loadRegistrationKey());
+
+        const dispatchedStateLoads = vi
+            .mocked(store.dispatch)
+            .mock.calls.filter(([action]) => action?.type === 'emhttp/load-single-state-file')
+            .map(([action]) => action.payload);
+
+        expect(dispatchedStateLoads).toEqual(Object.values(StateFileKey));
     });
 
-    it('reloads the devs state when devs.ini changes', async () => {
+    it('routes non-polled state files through the standard directory watcher', async () => {
         const { StateManager } = await import('@app/store/watch/state-watch.js');
         const { store } = await import('@app/store/index.js');
         const { loadSingleStateFile } = await import('@app/store/modules/emhttp.js');
 
-        StateManager.getInstance();
+        await StateManager.getInstance().ready;
+        vi.mocked(store.dispatch).mockClear();
 
-        const changeHandler = handlersByPath.get('/usr/local/emhttp/state/devs.ini')?.change;
+        const standardWatcher = watchRegistrations.find(
+            (registration) => registration.options.usePolling === false
+        );
+        const changeHandler = standardWatcher?.handlers.change;
         expect(changeHandler).toBeDefined();
 
         await changeHandler?.('/usr/local/emhttp/state/devs.ini');
 
         expect(store.dispatch).toHaveBeenCalledWith(loadSingleStateFile(StateFileKey.devs));
     });
 
-    it('reloads registration key when var.ini changes', async () => {
+    it('ignores non-state files while still allowing non-polled state files through the directory watcher', async () => {
+        const { StateManager } = await import('@app/store/watch/state-watch.js');
+
+        await StateManager.getInstance().ready;
+
+        const standardWatcher = watchRegistrations.find(
+            (registration) => registration.options.usePolling === false
+        );
+        const ignored = standardWatcher?.options.ignored;
+
+        expect(ignored).toBeTypeOf('function');
+        expect((ignored as (path: string) => boolean)('/usr/local/emhttp/state/README.txt')).toBe(true);
+        expect((ignored as (path: string) => boolean)('/usr/local/emhttp/state/devs.ini')).toBe(false);
+        expect((ignored as (path: string) => boolean)('/usr/local/emhttp/state/disks.ini')).toBe(true);
+    });
+
+    it('reloads registration key when var.ini is replaced after boot', async () => {
         const { StateManager } = await import('@app/store/watch/state-watch.js');
         const { store } = await import('@app/store/index.js');
         const { loadSingleStateFile } = await import('@app/store/modules/emhttp.js');
         const { loadRegistrationKey } = await import('@app/store/modules/registration.js');
 
-        StateManager.getInstance();
+        await StateManager.getInstance().ready;
+        vi.mocked(store.dispatch).mockClear();
 
-        const changeHandler = handlersByPath.get('/usr/local/emhttp/state/var.ini')?.change;
-        expect(changeHandler).toBeDefined();
+        const standardWatcher = watchRegistrations.find(
+            (registration) => registration.options.usePolling === false
+        );
+        const addHandler = standardWatcher?.handlers.add;
+        expect(addHandler).toBeDefined();
 
-        await changeHandler?.('/usr/local/emhttp/state/var.ini');
+        await addHandler?.('/usr/local/emhttp/state/var.ini');
 
         expect(store.dispatch).toHaveBeenNthCalledWith(1, loadSingleStateFile(StateFileKey.var));
         expect(store.dispatch).toHaveBeenNthCalledWith(2, loadRegistrationKey());
     });
+
+    it('routes polled state files through the polling directory watcher', async () => {
+        const { StateManager } = await import('@app/store/watch/state-watch.js');
+        const { store } = await import('@app/store/index.js');
+        const { loadSingleStateFile } = await import('@app/store/modules/emhttp.js');
+
+        await StateManager.getInstance().ready;
+        vi.mocked(store.dispatch).mockClear();
+
+        const pollingWatcher = watchRegistrations.find(
+            (registration) => registration.path === '/usr/local/emhttp/state/disks.ini'
+        );
+        const changeHandler = pollingWatcher?.handlers.change;
+        expect(changeHandler).toBeDefined();
+
+        await changeHandler?.('/usr/local/emhttp/state/disks.ini');
+
+        expect(store.dispatch).toHaveBeenCalledWith(loadSingleStateFile(StateFileKey.disks));
+    });
 });

api/src/index.ts

@@ -121,7 +121,9 @@ export const viteNodeApp = async (): Promise<NestFastifyApplication<RawServerDef
 
         // Start listening to file updates
         try {
-            StateManager.getInstance();
+            const timeout = budget.getTimeout(MAX_OPERATION_TIMEOUT_MS, BOOTSTRAP_RESERVED_MS);
+            const stateManager = StateManager.getInstance();
+            await withTimeout(stateManager.ready, timeout, 'stateManagerReady');
             logger.info('State manager initialized');
         } catch (error) {
             logger.error(error, 'Failed to initialize state manager');

api/src/store/watch/state-watch.ts

@@ -1,6 +1,6 @@
 import { join, parse } from 'path';
 
-import type { FSWatcher, FSWInstanceOptions } from 'chokidar';
+import type { ChokidarOptions, FSWatcher } from 'chokidar';
 import { watch } from 'chokidar';
 
 import { emhttpLogger } from '@app/core/log.js';
@@ -10,24 +10,40 @@ import { loadSingleStateFile } from '@app/store/modules/emhttp.js';
 import { loadRegistrationKey } from '@app/store/modules/registration.js';
 import { StateFileKey } from '@app/store/types.js';
 
-const chokidarOptionsForStateKey = (
-    key: StateFileKey
-): Partial<Pick<FSWInstanceOptions, 'usePolling' | 'interval'>> => {
-    if ([StateFileKey.disks, StateFileKey.shares].includes(key)) {
-        return {
-            usePolling: true,
-            interval: 10_000,
-        };
+const POLLED_STATE_KEYS = [StateFileKey.disks, StateFileKey.shares] as const;
+const POLLED_STATE_KEY_SET = new Set<StateFileKey>(POLLED_STATE_KEYS);
+const STATE_FILE_NAMES = new Set(Object.values(StateFileKey).map((key) => `${key}.ini`));
+
+const shouldIgnoreStatePath = (path: string): boolean => {
+    const parsed = parse(path);
+    const isStateFile = parsed.ext === '.ini' && STATE_FILE_NAMES.has(parsed.base);
+
+    if (!isStateFile) {
+        return true;
     }
-    return { usePolling: CHOKIDAR_USEPOLLING };
+
+    const stateFileKey = StateFileKey[parsed.name];
+    return POLLED_STATE_KEY_SET.has(stateFileKey);
 };
 
+const chokidarOptionsForStateDirectory = (): ChokidarOptions => ({
+    ignoreInitial: true,
+    ignored: (path, stats) => {
+        if (stats?.isDirectory()) {
+            return false;
+        }
+        return shouldIgnoreStatePath(path);
+    },
+    usePolling: CHOKIDAR_USEPOLLING,
+});
+
 export class StateManager {
     public static instance: StateManager | null = null;
+    public readonly ready: Promise<void>;
     private readonly fileWatchers: FSWatcher[] = [];
 
     private constructor() {
-        this.setupChokidarWatchForState();
+        this.ready = this.setupChokidarWatchForState();
     }
 
     public static getInstance(): StateManager {
@@ -43,6 +59,14 @@ export class StateManager {
         return StateFileKey[parsed.name];
     }
 
+    private async reloadStateFile(stateFile: StateFileKey, reason: 'add' | 'change' | 'startup-sync') {
+        emhttpLogger.debug('Loading state file for %s after %s', stateFile, reason);
+        await store.dispatch(loadSingleStateFile(stateFile));
+        if (stateFile === StateFileKey.var) {
+            await store.dispatch(loadRegistrationKey());
+        }
+    }
+
     private async handleStateFileUpdate(eventPath: string, event: 'add' | 'change') {
         const stateFile = this.getStateFileKeyFromPath(eventPath);
         if (!stateFile) {
@@ -51,11 +75,7 @@ export class StateManager {
         }
 
         try {
-            emhttpLogger.debug('Loading state file for %s after %s event', stateFile, event);
-            await store.dispatch(loadSingleStateFile(stateFile));
-            if (stateFile === StateFileKey.var) {
-                await store.dispatch(loadRegistrationKey());
-            }
+            await this.reloadStateFile(stateFile, event);
         } catch (error: unknown) {
             emhttpLogger.error(
                 'Failed to load state file: [%s] after %s event\nerror: %o',
@@ -66,15 +86,34 @@ export class StateManager {
         }
     }
 
-    private readonly setupChokidarWatchForState = () => {
+    private readonly reconcileStateAfterWatchSetup = async () => {
+        for (const stateFile of Object.values(StateFileKey)) {
+            await this.reloadStateFile(stateFile, 'startup-sync');
+        }
+    };
+
+    private readonly setupChokidarWatchForState = async () => {
         const { states } = getters.paths();
-        for (const key of Object.values(StateFileKey)) {
+
+        emhttpLogger.debug('Setting up watch for path: %s', states);
+        const directoryWatch = watch(states, chokidarOptionsForStateDirectory());
+        directoryWatch.on('add', async (path) => this.handleStateFileUpdate(path, 'add'));
+        directoryWatch.on('change', async (path) => this.handleStateFileUpdate(path, 'change'));
+        this.fileWatchers.push(directoryWatch);
+
+        for (const key of POLLED_STATE_KEYS) {
             const pathToWatch = join(states, `${key}.ini`);
             emhttpLogger.debug('Setting up watch for path: %s', pathToWatch);
-            const stateWatch = watch(pathToWatch, chokidarOptionsForStateKey(key));
+            const stateWatch = watch(pathToWatch, {
+                ignoreInitial: true,
+                usePolling: true,
+                interval: 10_000,
+            });
             stateWatch.on('add', async (path) => this.handleStateFileUpdate(path, 'add'));
             stateWatch.on('change', async (path) => this.handleStateFileUpdate(path, 'change'));
             this.fileWatchers.push(stateWatch);
         }
+
+        await this.reconcileStateAfterWatchSetup();
     };
 }