Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions crates/taskito-core/BINDING_CONTRACT.md
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,36 @@ Scheduler.handle_result ─▶ ResultOutcome ─▶ shell emits events / middlew
Channels: inbound `tokio::sync::mpsc::Receiver<Job>` (async); outbound
`crossbeam_channel::Sender<JobResult>` (sync, cloneable).

## Task errors (structured, cross-SDK)
When a task raises, the shell reports the failure as a **canonical JSON object**
serialized into `JobResult::Failure.error` (and thus into `jobs.error`,
`job_errors.error`, `dead_letter.error` — the storage layer never interprets it):

```json
{"errtype": "ValueError", "message": "bad value 42", "traceback": ["...frame...", "..."]}
```

- `errtype` — the exception's class name, as idiomatic per language (qualified
where the language has a notion of it). Required.
- `message` — the human-readable message, verbatim (keeps `error_like`
substring filters useful). Required, may be empty.
- `traceback` — array of strings, best-effort per shell; `[]` when the
language/runtime can't provide frames. Required key.

**Fallback rule (readers)**: an error string that does not parse as a JSON
object with a `message` key is a plain legacy/system string and MUST be
surfaced as-is. Core-generated maintenance errors (timeouts, worker-death
recovery, expiry, cancellation) remain plain strings by design.

**Retry semantics**: `retry_on`/`dont_retry_on`-style filtering matches on the
live exception object before formatting — the stored string never drives retry
decisions.

**Test vector** (assert byte-exact in each shell's formatter):
input errtype `BoomError`, message `it broke`, traceback `["frame1", "frame2"]` →
`{"errtype":"BoomError","message":"it broke","traceback":["frame1","frame2"]}`
(JSON with those three keys in that order, no extra whitespace).

## Types the shell produces / consumes
- **`Job`** — `job.rs`. Fields incl. `id`, `queue`, `task_name`, `payload: Vec<u8>` (opaque),
`status`, `priority`, `retry_count`, `max_retries`, `timeout_ms`, `unique_key`,
Expand Down
10 changes: 9 additions & 1 deletion crates/taskito-node/src/dispatcher.rs
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,15 @@ async fn run_one(callback: &TaskCallback, storage: &StorageBackend, mut job: Job
wall_time_ns,
}
} else {
failure(job, err.to_string(), wall_time_ns, false)
// `Error::to_string()` prepends the napi status ("GenericFailure, ");
// the bare reason is the JS error's string form, which the worker
// shapes into the cross-SDK structured-error JSON.
let reason = if err.reason.is_empty() {
err.to_string()
} else {
err.reason.clone()
};
failure(job, reason, wall_time_ns, false)
Comment thread
coderabbitai[bot] marked this conversation as resolved.
}
}
Ok(Err(_)) => failure(
Expand Down
15 changes: 2 additions & 13 deletions crates/taskito-python/src/native_async/task_executor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -165,19 +165,8 @@ fn run_task(py: Python<'_>, task_registry: &Py<PyAny>, job: &Job) -> PyResult<Op
}
}

fn format_python_error(py: Python<'_>, e: &PyErr) -> String {
if let Ok(tb_mod) = py.import("traceback") {
if let Ok(formatted) = tb_mod.call_method1(
"format_exception",
(e.get_type(py), e.value(py), e.traceback(py)),
) {
if let Ok(lines) = formatted.extract::<Vec<String>>() {
return lines.join("");
}
}
}
format!("{e}")
}
// One structured-error encoder for every worker path — see py_worker.rs.
use crate::py_worker::format_python_error;

fn is_cancelled_error(py: Python<'_>, e: &PyErr) -> bool {
if let Ok(exceptions_mod) = py.import("taskito.exceptions") {
Expand Down
14 changes: 13 additions & 1 deletion crates/taskito-python/src/py_worker.rs
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,19 @@ pub fn execute_task(
}

pub fn format_python_error(py: Python<'_>, e: &PyErr) -> String {
// Try to get a full traceback
// Canonical structured error (BINDING_CONTRACT.md "Task errors") — the
// Python module owns the encoding so every worker path emits one format.
if let Ok(errors_mod) = py.import("taskito.task_errors") {
if let Ok(encoded) = errors_mod.call_method1(
"encode_from_parts",
(e.get_type(py), e.value(py), e.traceback(py)),
) {
if let Ok(json) = encoded.extract::<String>() {
return json;
}
}
}
// Fallback: plain traceback text (readers treat non-JSON as legacy).
if let Ok(tb_mod) = py.import("traceback") {
if let Ok(formatted) = tb_mod.call_method1(
"format_exception",
Expand Down
1 change: 1 addition & 0 deletions dashboard/src/components/ui/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -103,5 +103,6 @@ export {
TableRow,
} from "./table";
export { Tabs, TabsContent, TabsList, TabsTrigger } from "./tabs";
export { TaskErrorBlock, TaskErrorSummary } from "./task-error";
export { Toaster, toast } from "./toaster";
export { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "./tooltip";
50 changes: 50 additions & 0 deletions dashboard/src/components/ui/task-error.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
import { cn } from "@/lib/cn";
import { parseTaskError, taskErrorSummary, taskErrorTooltip } from "@/lib/task-error";

/**
* Full task-error block for detail views. Structured errors get an
* `errtype: message` headline plus the traceback; legacy strings render
* verbatim. `className` carries per-site sizing/background (max-h, bg,
* padding) so each call site keeps its existing look.
*/
export function TaskErrorBlock({ error, className }: { error: string; className?: string }) {
const parsed = parseTaskError(error);

if (!parsed) {
return (
<pre
className={cn(
"overflow-auto whitespace-pre-wrap rounded-md font-mono text-[11px] text-danger",
className,
)}
>
{error}
</pre>
);
}

return (
<div className={cn("overflow-auto rounded-md font-mono text-[11px] text-danger", className)}>
<div className="font-semibold">{taskErrorSummary(parsed)}</div>
{parsed.traceback.length > 0 ? (
<pre className="mt-2 whitespace-pre-wrap">{parsed.traceback.join("\n")}</pre>
) : null}
</div>
);
}

/**
* One-line task-error for table cells: structured → `errtype: message` with
* the traceback tail in the title tooltip; legacy → the raw string with the
* full text as title (the pre-existing behavior).
*/
export function TaskErrorSummary({ error, className }: { error: string; className?: string }) {
const parsed = parseTaskError(error);
const text = parsed ? taskErrorSummary(parsed) : error;
const title = parsed ? taskErrorTooltip(parsed) : error;
return (
<span className={className} title={title}>
{text}
</span>
);
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { Link } from "@tanstack/react-router";
import { RotateCcw, Trash2 } from "lucide-react";
import { Badge, Button } from "@/components/ui";
import { Badge, Button, TaskErrorBlock } from "@/components/ui";
import type { DeadLetter } from "@/lib/api-types";
import { formatRelative } from "@/lib/time";
import { useDeleteDeadLetter, useRetryDeadLetter } from "../hooks";
Expand Down Expand Up @@ -39,9 +39,7 @@ export function DeadLetterRow({ item }: DeadLetterRowProps) {
</span>
</div>
{item.error ? (
<pre className="mt-2 max-h-32 overflow-auto whitespace-pre-wrap rounded-md bg-danger-dim/30 p-2 font-mono text-[11px] text-danger">
{item.error}
</pre>
<TaskErrorBlock error={item.error} className="mt-2 max-h-32 bg-danger-dim/30 p-2" />
) : null}
</div>
<div className="flex gap-2">
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import {
TableHead,
TableHeader,
TableRow,
TaskErrorSummary,
} from "@/components/ui";
import type { DeadLetter } from "@/lib/api-types";
import { formatRelative } from "@/lib/time";
Expand Down Expand Up @@ -60,9 +61,10 @@ function DeadLetterTableRow({ item }: { item: DeadLetter }) {
<TableCell className="text-[var(--fg-muted)]">{item.queue}</TableCell>
<TableCell className="max-w-[300px]">
{item.error ? (
<span className="block truncate font-mono text-[0.78rem] text-danger" title={item.error}>
{item.error}
</span>
<TaskErrorSummary
error={item.error}
className="block truncate font-mono text-[0.78rem] text-danger"
/>
) : (
<span className="text-[var(--fg-subtle)]">—</span>
)}
Expand Down
14 changes: 14 additions & 0 deletions dashboard/src/features/dead-letters/utils.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,19 @@ const TRACEBACK_CONNECTION_ERROR = `Traceback (most recent call last):
raise ConnectionError(f"upstream {endpoint} unreachable")
ConnectionError: upstream /v1/users unreachable`;

const STRUCTURED_ERROR =
'{"errtype":"BoomError","message":"it broke","traceback":["frame1","frame2"]}';

describe("extractExceptionClass", () => {
it("pulls the class name from a standard traceback", () => {
expect(extractExceptionClass(TRACEBACK_VALUE_ERROR)).toBe("ValueError");
expect(extractExceptionClass(TRACEBACK_CONNECTION_ERROR)).toBe("ConnectionError");
});

it("uses errtype for structured errors", () => {
expect(extractExceptionClass(STRUCTURED_ERROR)).toBe("BoomError");
});

it("handles custom exception class names", () => {
const traceback = "something\nMyCustomError: oops";
expect(extractExceptionClass(traceback)).toBe("MyCustomError");
Expand All @@ -61,6 +68,13 @@ describe("extractReason", () => {
expect(extractReason(null)).toMatch(/no error captured/i);
expect(extractReason(" ")).toMatch(/no error captured/i);
});

it("uses message for structured errors", () => {
expect(extractReason(STRUCTURED_ERROR)).toBe("it broke");
expect(extractReason('{"errtype":"E","message":"","traceback":[]}')).toMatch(
/no error captured/i,
);
});
});

describe("groupByError", () => {
Expand Down
17 changes: 12 additions & 5 deletions dashboard/src/features/dead-letters/utils.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import type { DeadLetter } from "@/lib/api-types";
import { parseTaskError } from "@/lib/task-error";

export interface DeadLetterGroup {
/** Stable key: `taskName::ExceptionClass`. */
Expand All @@ -21,14 +22,17 @@ const EMPTY_ERROR_LABEL = "(no error captured)";
const UNKNOWN_EXCEPTION = "Error";

/**
* Extract the exception class from a Python traceback string.
* Extract the exception class from an error string.
*
* Python tracebacks end with a line like ``ValueError: permanent failure…``.
* We read the last non-blank line and pull the token before the first `:`.
* Falls back to "Error" if the input isn't a recognisable traceback.
* Structured errors (canonical JSON per the cross-SDK contract) carry the
* class in `errtype`. Legacy Python tracebacks end with a line like
* ``ValueError: permanent failure…`` — we read the last non-blank line and
* pull the token before the first `:`. Falls back to "Error" otherwise.
*/
export function extractExceptionClass(error: string | null | undefined): string {
if (!error) return UNKNOWN_EXCEPTION;
const structured = parseTaskError(error);
if (structured) return structured.errtype;
const trimmed = error.trim();
if (!trimmed) return UNKNOWN_EXCEPTION;
const lines = trimmed.split("\n");
Expand All @@ -47,10 +51,13 @@ export function extractExceptionClass(error: string | null | undefined): string

/**
* Extract the one-line reason from a traceback — the text after the exception
* class on the final traceback line. Useful for a group's sub-summary.
* class on the final traceback line (or `message` for structured errors).
* Useful for a group's sub-summary.
*/
export function extractReason(error: string | null | undefined): string {
if (!error) return EMPTY_ERROR_LABEL;
const structured = parseTaskError(error);
if (structured) return structured.message || EMPTY_ERROR_LABEL;
const trimmed = error.trim();
if (!trimmed) return EMPTY_ERROR_LABEL;
const lines = trimmed.split("\n");
Expand Down
6 changes: 2 additions & 4 deletions dashboard/src/features/jobs/components/job-errors-tab.tsx
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { AlertOctagon } from "lucide-react";
import { EmptyState, ErrorState, Skeleton } from "@/components/ui";
import { EmptyState, ErrorState, Skeleton, TaskErrorBlock } from "@/components/ui";
import type { JobError } from "@/lib/api-types";
import { formatAbsolute } from "@/lib/time";

Expand Down Expand Up @@ -44,9 +44,7 @@ export function JobErrorsTab({ errors, loading, error, onRetry }: JobErrorsTabPr
{formatAbsolute(err.failed_at)}
</span>
</div>
<pre className="max-h-60 overflow-auto whitespace-pre-wrap rounded-md bg-danger-dim/30 p-3 font-mono text-[11px] text-danger">
{err.error}
</pre>
<TaskErrorBlock error={err.error} className="max-h-60 bg-danger-dim/30 p-3" />
</div>
))}
</div>
Expand Down
13 changes: 9 additions & 4 deletions dashboard/src/features/jobs/components/job-overview-tab.tsx
Original file line number Diff line number Diff line change
@@ -1,6 +1,13 @@
import { ExternalLink as ExternalLinkIcon } from "lucide-react";
import { type ReactNode, useMemo } from "react";
import { buttonVariants, Card, CardContent, CardHeader, CardTitle } from "@/components/ui";
import {
buttonVariants,
Card,
CardContent,
CardHeader,
CardTitle,
TaskErrorBlock,
} from "@/components/ui";
import { applyJobContext, useIntegrations } from "@/features/settings";
import type { Job } from "@/lib/api-types";
import { cn } from "@/lib/cn";
Expand Down Expand Up @@ -99,9 +106,7 @@ export function JobOverviewTab({ job }: JobOverviewTabProps) {
<CardTitle>Last error</CardTitle>
</CardHeader>
<CardContent>
<pre className="max-h-80 overflow-auto whitespace-pre-wrap rounded-md bg-danger-dim/40 p-3 font-mono text-[11px] text-danger">
{job.error}
</pre>
<TaskErrorBlock error={job.error} className="max-h-80 bg-danger-dim/40 p-3" />
</CardContent>
</Card>
) : null}
Expand Down
8 changes: 5 additions & 3 deletions dashboard/src/features/jobs/components/job-table.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import {
ErrorState,
StatusBadge,
TableSkeleton,
TaskErrorSummary,
Tooltip,
TooltipContent,
TooltipTrigger,
Expand Down Expand Up @@ -99,9 +100,10 @@ export function JobTable({ jobs, loading, error, onRetry }: JobTableProps) {
const err = getValue<string | null>();
if (!err) return <span className="text-[var(--fg-subtle)]">—</span>;
return (
<span className="line-clamp-1 max-w-[320px] text-xs text-danger" title={err}>
{err}
</span>
<TaskErrorSummary
error={err}
className="line-clamp-1 max-w-[320px] text-xs text-danger"
/>
);
},
});
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { Link } from "@tanstack/react-router";
import { Layers } from "lucide-react";
import { Badge, EmptyState } from "@/components/ui";
import { Badge, EmptyState, TaskErrorSummary } from "@/components/ui";
import type { WorkflowNode } from "@/lib/api-types";
import { WORKFLOW_NODE_LABEL, WORKFLOW_NODE_TONE } from "@/lib/status";
import { formatDuration, formatRelative } from "@/lib/time";
Expand Down Expand Up @@ -76,7 +76,7 @@ function NodeRow({ node }: { node: WorkflowNode }) {
{duration ?? "—"}
</td>
<td className="max-w-[200px] truncate px-4 py-3 text-[0.8rem] text-danger">
{node.error ?? ""}
{node.error ? <TaskErrorSummary error={node.error} /> : ""}
</td>
</tr>
);
Expand Down
Loading