Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 49 additions & 3 deletions src/orchestrator/factory.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,24 @@ class EscalatingTriage extends StaticTriage {
}
}

class SlackClarifiedTriage extends EscalatingTriage {
override async triage(issue: LinearIssue): Promise<TriageDecision> {
if (issue.description.includes('Human clarification from Slack:')) {
const decision = await super.triage({
...issue,
description: `${issue.description}\nImplement the clarified behavior and verify it with tests.`,
})
return {
...decision,
thin: false,
confidence: 'high',
rationale: 'Human Slack clarification supplied enough acceptance detail.',
}
}
return super.triage(issue)
}
}

class SpawnFailingFleetClient extends FakeFleetClient {
override async spawn(input: SpawnInput): Promise<SpawnResult> {
this.spawns.push(input)
Expand Down Expand Up @@ -6782,7 +6800,7 @@ describe('FactoryLoop', () => {
const factory = createFactory(config({ slack: slackConfig() }), {
mount,
fleet,
triage: new EscalatingTriage({ rationale: 'No repository route matched.' }),
triage: new EscalatingTriage({ rationale: 'Matched repository from Linear label.' }),
slack,
})

Expand All @@ -6794,11 +6812,39 @@ describe('FactoryLoop', () => {
const slackRoots = mount.writes.filter((write) => isSlackRootWritePath(write.path))
expect(slackRoots).toHaveLength(1)
expect((slackRoots[0]?.content as { text?: string }).text).toContain('AR-20: factory triage escalation for [factory-e2e] Fix factory issue 20')
expect((slackRoots[0]?.content as { text?: string }).text).toContain('Reason: low-confidence triage and thin issue context: No repository route matched.')
expect((slackRoots[0]?.content as { text?: string }).text).toContain('Question: Please clarify')
expect((slackRoots[0]?.content as { text?: string }).text).toContain('Reason: low-confidence triage and thin issue context: Matched repository from Linear label.')
expect((slackRoots[0]?.content as { text?: string }).text).toContain('Question: Factory matched AgentWorkforce/pear. Please clarify the concrete expected behavior, constraints, and acceptance criteria/tests before dispatch.')
expect(factory.status().counters.errors).toBeUndefined()
expect(factory.status().counters.triageEscalations).toBe(1)
expect(slack.roots).toEqual([])
})

it('uses a human Slack answer to retry pre-dispatch triage and dispatch when clarified', async () => {
const mount = new CloudWritebackFakeMountClient({ [issuePath(23)]: issueFile(23) })
const fleet = new FakeFleetClient()
const factory = createFactory(config({ slack: slackConfig() }), {
mount,
fleet,
triage: new SlackClarifiedTriage({ rationale: 'Matched repository from Linear label.' }),
})

await factory.runOnce()
expect(fleet.spawns).toEqual([])
expect(factory.status().counters.triageEscalations).toBe(1)

emitSlackReply(mount, slackReplyFixturePath('C0FACTORY__factory-e2e', mount.threadTs, 'human-clarifies-23'), 'slack-human-clarifies-23', {
text: 'When deployed via ./workforce, one-click deploy in cloud should auto-join the configured Slack channel and ask there if blocked. Verify with tests.',
user: 'U123',
user_is_bot: false,
})

await vi.waitFor(() => expect(factory.status().counters.slackTriageAnswersDispatched).toBe(1))
expect(fleet.spawns.map((spawn) => spawn.name)).toEqual(['ar-23-impl-pear', 'ar-23-review'])
expect(factory.status().counters.slackTriageAnswersDispatched).toBe(1)
expect(factory.status().counters.slackTriageAnswersStillEscalated).toBeUndefined()
Comment thread
coderabbitai[bot] marked this conversation as resolved.
expect(factory.status().counters.errors).toBeUndefined()
})

it('ignores a human Slack thread reply after the issue has no in-flight implementer', async () => {
const mount = new CloudWritebackFakeMountClient({ [issuePath(21)]: issueFile(21) })
const fleet = new FakeFleetClient()
Expand Down
111 changes: 104 additions & 7 deletions src/orchestrator/factory.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1317,7 +1317,7 @@ export class FactoryLoop implements Factory {
const escalationReason = triageEscalationReason(decision)
if (escalationReason) {
await this.#escalateTriageToSlack(decision, escalationReason, dryRun)
this.#error(new Error(`${escalationReason}; escalation required`), decision.issue)
this.#recordTriageEscalation(decision, escalationReason)
return { issue: decision.issue, agents: [], dryRun }
}

Expand Down Expand Up @@ -1521,7 +1521,7 @@ export class FactoryLoop implements Factory {
const escalationReason = triageEscalationReason(decision)
if (escalationReason) {
await this.#escalateTriageToSlack(decision, escalationReason, this.#config.dryRun)
this.#error(new Error(`${escalationReason}; escalation required`), decision.issue)
this.#recordTriageEscalation(decision, escalationReason)
return
}

Expand Down Expand Up @@ -3156,6 +3156,14 @@ export class FactoryLoop implements Factory {
this.#counters[name] = (this.#counters[name] ?? 0) + 1
}

#recordTriageEscalation(decision: TriageDecision, reason: string): void {
this.#increment('triageEscalations')
this.#logger.warn?.('[factory] triage escalation required', {
issue: decision.issue,
reason,
})
}

async #shouldSkipSlackWriteback(context: string): Promise<boolean> {
if (!this.#config.slack) return false

Expand Down Expand Up @@ -3439,7 +3447,7 @@ export class FactoryLoop implements Factory {
text: [
`${decision.issue.key}: factory triage escalation for ${issue?.title ?? decision.issue.key}`,
`Reason: ${reason}`,
`Question: Please clarify the intended repo/approach or add enough acceptance detail for the factory agent to proceed.`,
`Question: ${triageEscalationQuestion(decision)}`,
].join('\n'),
})
await this.#state.setSlackThread(this.#workspaceId, issueKey(decision.issue), root.threadId)
Expand Down Expand Up @@ -3644,19 +3652,27 @@ export class FactoryLoop implements Factory {
}

async #routeSlackAnswerToImplementers(record: InFlightIssue, reply: SlackReply): Promise<void> {
if (!this.#config.slack || !this.#fleet.sendInput) {
if (!this.#config.slack) {
return
}

const text = reply.text.trim()
if (!text) {
this.#increment('slackAnswersIgnoredEmpty')
return
}

const liveRecord = (await this.#batch()).getIssue(record.issue)
if (!liveRecord || liveRecord.dryRun) {
if (isTriageEscalationWatchRecord(record)) {
await this.#handleTriageEscalationSlackAnswer(record, text)
return
}
this.#increment('slackAnswersIgnoredNoInFlight')
return
}

const text = reply.text.trim()
if (!text) {
this.#increment('slackAnswersIgnoredEmpty')
if (!this.#fleet.sendInput) {
return
}

Expand All @@ -3678,6 +3694,54 @@ export class FactoryLoop implements Factory {
}
}

async #handleTriageEscalationSlackAnswer(record: InFlightIssue, text: string): Promise<void> {
const issue = await this.#readIssue(record.issue.path)
if (!issue || !isInFactoryScope(issue, this.#config.safety) || !isRealLinearIssue(issue)) {
this.#increment('slackTriageAnswersIgnoredIssueUnavailable')
return
}
if (!this.#states.isRole(issue.stateId, 'readyForAgent')) {
this.#increment('slackTriageAnswersIgnoredIssueNotReady')
return
}

const batch = await this.#batch()
if (batch.isInFlight(record.issue) || batch.isQueued(record.issue)) {
this.#increment('slackTriageAnswersIgnoredAlreadyActive')
return
}
if (await this.#dispatchBlockReason(record.issue)) {
this.#increment('slackTriageAnswersIgnoredBlocked')
return
}

const clarifiedIssue = issueWithSlackClarification(issue, text)
const decision = await this.#triage.triage(clarifiedIssue, {
config: this.#config,
repoMap: repoMapFromConfig(this.#config),
})
const escalationReason = triageEscalationReason(decision)
if (escalationReason) {
this.#increment('slackTriageAnswersStillEscalated')
this.#logger.warn?.('[factory] Slack triage answer still leaves issue escalated', {
issue: record.issue,
reason: escalationReason,
})
return
}

if (batch.canStart()) {
await this.dispatch(decision, { dryRun: this.#config.dryRun })
this.#increment('slackTriageAnswersDispatched')
return
}

if (batch.queue(decision, this.#config.dryRun)) {
this.#increment('slackTriageAnswersQueued')
this.#emit('issue-queued', { issue: decision.issue })
}
}

// Inject the human's Slack reply into the agent framed as the
// <integration-event> the spawn prompt tells it to expect (not an ambiguous
// "Slack reply for ..." keystroke), so the agent recognizes it as the awaited
Expand Down Expand Up @@ -5271,6 +5335,39 @@ const triageEscalationReason = (decision: TriageDecision): string | undefined =>
return `${reasons.join(' and ')}${decision.rationale ? `: ${decision.rationale}` : ''}`
}

const triageEscalationQuestion = (decision: TriageDecision): string => {
const routedRepos = decision.routes.map((route) => route.repo).filter(Boolean)
if (routedRepos.length === 0) {
return [
'Which repository or repositories should handle this issue?',
'Please include the intended approach and the acceptance criteria/tests the agent should satisfy.',
].join(' ')
}
if (decision.thin) {
return [
`Factory matched ${routedRepos.join(', ')}.`,
'Please clarify the concrete expected behavior, constraints, and acceptance criteria/tests before dispatch.',
].join(' ')
}
return [
`Factory matched ${routedRepos.join(', ')}, but triage confidence is low.`,
'Please confirm the intended repo/approach or correct the route before dispatch.',
].join(' ')
}

const isTriageEscalationWatchRecord = (record: InFlightIssue): boolean =>
record.agents.size === 0 && record.invocationIds.size === 0 && triageEscalationReason(record.decision) !== undefined

const issueWithSlackClarification = (issue: LinearIssue, text: string): LinearIssue => ({
...issue,
description: [
issue.description,
'',
'Human clarification from Slack:',
text,
].join('\n'),
})

const slackAnswerInput = (issue: IssueRef, text: string): string =>
`Slack reply for ${issue.key}:\n${text}\r`

Expand Down