Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 61 additions & 4 deletions apps/server/src/vcs/VcsStatusBroadcaster.test.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
import { assert, it, describe } from "@effect/vitest";
import * as NodeServices from "@effect/platform-node/NodeServices";
import * as Cause from "effect/Cause";
import * as Deferred from "effect/Deferred";
import * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
import * as Exit from "effect/Exit";
import * as FileSystem from "effect/FileSystem";
import * as Layer from "effect/Layer";
import * as Logger from "effect/Logger";
import * as Option from "effect/Option";
import * as Path from "effect/Path";
import * as Scope from "effect/Scope";
Expand Down Expand Up @@ -432,6 +434,12 @@ describe("VcsStatusBroadcaster", () => {
remoteInvalidationCalls: 0,
remoteStatusRefreshUpstreamValues: [] as Array<boolean | undefined>,
};
const privateCwd = "/private/user/workspace/repo";
const nestedCause = new Error("private nested VCS failure");
const messages: Array<ReadonlyArray<unknown>> = [];
const logger = Logger.make<unknown, void>(({ message }) => {
messages.push(message as ReadonlyArray<unknown>);
});
let firstRemoteAttemptDeferred: Deferred.Deferred<void> | null = null;
const testLayer = VcsStatusBroadcaster.layer.pipe(
Layer.provideMerge(NodeServices.layer),
Expand All @@ -450,8 +458,9 @@ describe("VcsStatusBroadcaster", () => {
return Effect.fail(
new GitManagerError({
operation: "VcsStatusBroadcaster.test",
cwd: "/repo",
detail: "initial remote status failed",
cwd: privateCwd,
detail: "private initial remote status failure",
cause: nestedCause,
}),
).pipe(
Effect.ensuring(
Expand Down Expand Up @@ -482,7 +491,7 @@ describe("VcsStatusBroadcaster", () => {
const remoteUpdatedDeferred = yield* Deferred.make<VcsStatusStreamEvent>();
yield* Stream.runForEach(
broadcaster.streamStatus(
{ cwd: "/repo" },
{ cwd: privateCwd },
{ automaticRemoteRefreshInterval: Effect.succeed(Duration.zero) },
),
(event) =>
Expand All @@ -494,6 +503,24 @@ describe("VcsStatusBroadcaster", () => {
yield* Deferred.await(firstRemoteAttemptDeferred);
yield* Effect.yieldNow;
assert.equal(state.remoteStatusCalls, 1);
assert.deepStrictEqual(
messages.find((message) => message[0] === "VCS remote status refresh failed"),
[
"VCS remote status refresh failed",
{
cwdLength: privateCwd.length,
reasonCount: 1,
failureCount: 1,
failureTags: ["GitManagerError"],
failureOperations: ["VcsStatusBroadcaster.test"],
defectCount: 0,
defectTags: [],
interruptionCount: 0,
consecutiveFailures: 1,
nextDelayMs: 30_000,
},
],
);

yield* TestClock.adjust(Duration.seconds(30));
const remoteUpdated = yield* Deferred.await(remoteUpdatedDeferred);
Expand All @@ -507,7 +534,15 @@ describe("VcsStatusBroadcaster", () => {
assert.deepStrictEqual(state.remoteStatusRefreshUpstreamValues, [false, false]);

yield* Scope.close(scope, Exit.void);
}).pipe(Effect.provide(Layer.merge(testLayer, TestClock.layer())));
}).pipe(
Effect.provide(
Layer.mergeAll(
testLayer,
TestClock.layer(),
Logger.layer([logger], { mergeWithExisting: false }),
),
),
);
});

it.effect("delays automatic refresh when a cached remote snapshot is available", () => {
Expand Down Expand Up @@ -575,6 +610,28 @@ describe("VcsStatusBroadcaster", () => {
);
});

it("summarizes refresh causes without exposing nested failure details", () => {
const nestedCause = new Error("private nested failure detail");
const failure = new GitManagerError({
operation: "VcsStatusBroadcaster.remoteStatus",
cwd: "/private/user/workspace/repo",
detail: "private Git failure detail",
cause: nestedCause,
});
const cause = Cause.combine(Cause.fail(failure), Cause.die(new TypeError("private defect")));

assert.deepStrictEqual(VcsStatusBroadcaster.remoteRefreshFailureDiagnostics(cause), {
reasonCount: 2,
failureCount: 1,
failureTags: ["GitManagerError"],
failureOperations: ["VcsStatusBroadcaster.remoteStatus"],
defectCount: 1,
defectTags: ["TypeError"],
interruptionCount: 0,
});
assert.strictEqual(failure.cause, nestedCause);
});

it.effect("stops the remote poller after the last stream subscriber disconnects", () => {
const state = {
currentLocalStatus: baseLocalStatus,
Expand Down
95 changes: 93 additions & 2 deletions apps/server/src/vcs/VcsStatusBroadcaster.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import * as Cause from "effect/Cause";
import * as Context from "effect/Context";
import * as Duration from "effect/Duration";
import * as Effect from "effect/Effect";
Expand Down Expand Up @@ -26,6 +27,91 @@ import * as GitWorkflowService from "../git/GitWorkflowService.ts";
const DEFAULT_VCS_STATUS_REFRESH_INTERVAL = Duration.seconds(30);
const VCS_STATUS_REFRESH_FAILURE_BASE_DELAY = Duration.seconds(30);
const VCS_STATUS_REFRESH_FAILURE_MAX_DELAY = Duration.minutes(15);
const MAX_FAILURE_DIAGNOSTIC_VALUES = 8;
const MAX_FAILURE_DIAGNOSTIC_VALUE_LENGTH = 128;

function boundedDiagnosticValue(value: string): string {
return value.slice(0, MAX_FAILURE_DIAGNOSTIC_VALUE_LENGTH);
}

function diagnosticValueTag(value: unknown): string {
try {
if (
typeof value === "object" &&
value !== null &&
"_tag" in value &&
typeof value._tag === "string"
) {
return boundedDiagnosticValue(value._tag);
}
if (value instanceof Error) {
return boundedDiagnosticValue(value.name);
}
return typeof value;
} catch {
return "Uninspectable";
}
}

function diagnosticFailureOperation(value: unknown): string | undefined {
try {
if (
typeof value === "object" &&
value !== null &&
"operation" in value &&
typeof value.operation === "string"
) {
return boundedDiagnosticValue(value.operation);
}
} catch {
return undefined;
}
return undefined;
}

function addUniqueDiagnosticValue(values: Array<string>, value: string | undefined): void {
if (
value !== undefined &&
values.length < MAX_FAILURE_DIAGNOSTIC_VALUES &&
!values.includes(value)
) {
values.push(value);
}
}

export function remoteRefreshFailureDiagnostics(cause: Cause.Cause<unknown>) {
const failureTags: Array<string> = [];
const failureOperations: Array<string> = [];
const defectTags: Array<string> = [];
let failureCount = 0;
let defectCount = 0;
let interruptionCount = 0;

for (const reason of cause.reasons) {
if (Cause.isFailReason(reason)) {
failureCount += 1;
addUniqueDiagnosticValue(failureTags, diagnosticValueTag(reason.error));
addUniqueDiagnosticValue(failureOperations, diagnosticFailureOperation(reason.error));
continue;
}
if (Cause.isDieReason(reason)) {
defectCount += 1;
addUniqueDiagnosticValue(defectTags, diagnosticValueTag(reason.defect));
continue;
}
interruptionCount += 1;
}

return {
reasonCount: cause.reasons.length,
failureCount,
failureTags,
failureOperations,
defectCount,
defectTags,
interruptionCount,
};
}

interface VcsStatusChange {
readonly cwd: string;
Expand Down Expand Up @@ -318,14 +404,19 @@ export const make = Effect.gen(function* () {
return activeInterval;
}

const interruptionReasons = exit.cause.reasons.filter(Cause.isInterruptReason);
if (interruptionReasons.length > 0) {
return yield* Effect.failCause(Cause.fromReasons<never>(interruptionReasons));
}

const consecutiveFailures = yield* Ref.updateAndGet(
consecutiveFailuresRef,
(count) => count + 1,
);
const nextDelay = remoteRefreshFailureDelay(consecutiveFailures, activeInterval);
yield* Effect.logWarning("VCS remote status refresh failed", {
cwd,
detail: exit.cause.toString(),
cwdLength: cwd.length,
...remoteRefreshFailureDiagnostics(exit.cause),
consecutiveFailures,
nextDelayMs: Duration.toMillis(nextDelay),
});
Expand Down
Loading