diff --git a/airflow-core/src/airflow/ui/public/i18n/locales/en/components.json b/airflow-core/src/airflow/ui/public/i18n/locales/en/components.json index 10351910eee16..837726e376dc6 100644 --- a/airflow-core/src/airflow/ui/public/i18n/locales/en/components.json +++ b/airflow-core/src/airflow/ui/public/i18n/locales/en/components.json @@ -65,6 +65,7 @@ "lastDagRun_other": "Last {{count}} Dag Runs", "lastTaskInstance_one": "Last Task Instance", "lastTaskInstance_other": "Last {{count}} Task Instances", + "medianTotalDuration": "Total: {{duration}}", "queuedDuration": "Queued Duration", "runAfter": "Run After", "runDuration": "Run Duration" diff --git a/airflow-core/src/airflow/ui/src/components/DurationChart.tsx b/airflow-core/src/airflow/ui/src/components/DurationChart.tsx index 8008307fabf40..b558249ff1d9f 100644 --- a/airflow-core/src/airflow/ui/src/components/DurationChart.tsx +++ b/airflow-core/src/airflow/ui/src/components/DurationChart.tsx @@ -25,9 +25,9 @@ import { LineElement, BarElement, Filler, + Legend, Tooltip, } from "chart.js"; -import type { PartialEventContext } from "chartjs-plugin-annotation"; import annotationPlugin from "chartjs-plugin-annotation"; import dayjs from "dayjs"; import { Bar } from "react-chartjs-2"; @@ -37,8 +37,15 @@ import { useNavigate } from "react-router-dom"; import type { TaskInstanceResponse, GridRunsResponse } from "openapi/requests/types.gen"; import { useTimezone } from "src/context/timezone"; import { getComputedCSSVariableValue } from "src/theme"; -import { DEFAULT_DATETIME_FORMAT, formatDate, renderDuration } from "src/utils/datetimeUtils"; +import { + DEFAULT_DATETIME_FORMAT, + formatDate, + getDurationTickStep, + renderCompactDuration, + renderDuration, +} from "src/utils/datetimeUtils"; import { buildTaskInstanceUrl } from "src/utils/links"; +import { median } from "src/utils/median"; ChartJS.register( CategoryScale, @@ -47,15 +54,12 @@ ChartJS.register( BarElement, LineElement, Filler, + Legend, Tooltip, annotationPlugin, ); -const average = (ctx: PartialEventContext, index: number) => { - const values: Array | undefined = ctx.chart.data.datasets[index]?.data as Array | undefined; - - return values === undefined ? 0 : values.reduce((initial, next) => initial + next, 0) / values.length; -}; +const CHART_HEIGHT = "280px"; type RunResponse = GridRunsResponse | TaskInstanceResponse; @@ -70,6 +74,24 @@ const getDuration = (start: string, end: string | null) => { return dayjs.duration(endDate.diff(startDate)).asSeconds(); }; +const getQueuedDuration = (entry: RunResponse, kind: "Dag Run" | "Task Instance") => { + if (kind === "Dag Run") { + const run = entry as GridRunsResponse; + + return run.queued_at !== null && run.start_date !== null && run.queued_at < run.start_date + ? getDuration(run.queued_at, run.start_date) + : 0; + } + + const taskInstance = entry as TaskInstanceResponse; + + return taskInstance.queued_when !== null && + taskInstance.start_date !== null && + taskInstance.queued_when < taskInstance.start_date + ? getDuration(taskInstance.queued_when, taskInstance.start_date) + : 0; +}; + const getTickLabelFormat = (entries: Array): string => { if (entries.length < 2) { return "HH:mm:ss"; @@ -121,28 +143,28 @@ export const DurationChart = ({ } }); - const runAnnotation = { - borderColor: "grey", - borderWidth: 1, - label: { - content: (ctx: PartialEventContext) => renderDuration(average(ctx, 1), false) ?? "0", - display: true, - position: "end", - }, - scaleID: "y", - value: (ctx: PartialEventContext) => average(ctx, 1), - }; + const queuedDurations = entries.map((entry) => getQueuedDuration(entry, kind)); + const runDurations = entries.map((entry) => + entry.start_date === null ? 0 : getDuration(entry.start_date, entry.end_date), + ); + // Bars stack queued under run, so the reference line tracks the same total the + // reader sees at the top of each bar. + const totalDurations = runDurations.map((duration, index) => duration + (queuedDurations[index] ?? 0)); + const medianTotal = median(totalDurations); - const queuedAnnotation = { + const medianAnnotation = { borderColor: "grey", + borderDash: [6, 4], borderWidth: 1, label: { - content: (ctx: PartialEventContext) => renderDuration(average(ctx, 0), false) ?? "0", + content: translate("durationChart.medianTotalDuration", { + duration: renderCompactDuration(medianTotal), + }), display: true, - position: "end", + position: "start", }, scaleID: "y", - value: (ctx: PartialEventContext) => average(ctx, 0), + value: medianTotal, }; return ( @@ -152,134 +174,122 @@ export const DurationChart = ({ ? translate("durationChart.lastDagRun", { count: entries.length }) : translate("durationChart.lastTaskInstance", { count: entries.length })} - { - switch (kind) { - case "Dag Run": { - const run = entry as GridRunsResponse; - - return run.queued_at !== null && run.start_date !== null && run.queued_at < run.start_date - ? Number(getDuration(run.queued_at, run.start_date)) - : 0; - } - case "Task Instance": { - const taskInstance = entry as TaskInstanceResponse; - - return taskInstance.queued_when !== null && - taskInstance.start_date !== null && - taskInstance.queued_when < taskInstance.start_date - ? Number(getDuration(taskInstance.queued_when, taskInstance.start_date)) - : 0; - } - default: - return 0; - } - }), - label: translate("durationChart.queuedDuration"), - }, - { - backgroundColor: entries.map( - (entry: RunResponse) => - (entry.state ? stateColorMap[entry.state] : undefined) ?? "oklch(0.5 0 0)", - ), - data: entries.map((entry: RunResponse) => - entry.start_date === null ? 0 : Number(getDuration(entry.start_date, entry.end_date)), - ), - label: translate("durationChart.runDuration"), - }, - ], - labels: entries.map((entry: RunResponse) => dayjs(entry.run_after).format(DEFAULT_DATETIME_FORMAT)), - }} - datasetIdKey="id" - options={{ - animation: isAutoRefreshing ? false : undefined, - onClick: (_event, elements) => { - const [element] = elements; - - if (!element) { - return; - } - - switch (kind) { - case "Dag Run": { - const entry = entries[element.index] as GridRunsResponse | undefined; - const baseUrl = `/dags/${entry?.dag_id}/runs/${entry?.run_id}`; - - void Promise.resolve(navigate(baseUrl)); - break; + {/* Height is fixed because the chart now flexes horizontally: with Chart.js' + default 2:1 aspect ratio a wide monitor would otherwise scale it past + 1000px tall. */} + + + (entry.state ? stateColorMap[entry.state] : undefined) ?? "oklch(0.5 0 0)", + ), + data: runDurations, + label: translate("durationChart.runDuration"), + }, + ], + labels: entries.map((entry: RunResponse) => + dayjs(entry.run_after).format(DEFAULT_DATETIME_FORMAT), + ), + }} + datasetIdKey="id" + options={{ + animation: isAutoRefreshing ? false : undefined, + maintainAspectRatio: false, + onClick: (_event, elements) => { + const [element] = elements; + + if (!element) { + return; } - case "Task Instance": { - const entry = entries[element.index] as TaskInstanceResponse | undefined; - if (entry === undefined) { + switch (kind) { + case "Dag Run": { + const entry = entries[element.index] as GridRunsResponse | undefined; + const baseUrl = `/dags/${entry?.dag_id}/runs/${entry?.run_id}`; + + void Promise.resolve(navigate(baseUrl)); break; } + case "Task Instance": { + const entry = entries[element.index] as TaskInstanceResponse | undefined; + + if (entry === undefined) { + break; + } - const baseUrl = buildTaskInstanceUrl({ - currentPathname: location.pathname, - dagId: entry.dag_id, - isMapped: entry.map_index >= 0, - mapIndex: entry.map_index.toString(), - runId: entry.dag_run_id, - taskId: entry.task_id, - }); - - void Promise.resolve(navigate(baseUrl)); - break; + const baseUrl = buildTaskInstanceUrl({ + currentPathname: location.pathname, + dagId: entry.dag_id, + isMapped: entry.map_index >= 0, + mapIndex: entry.map_index.toString(), + runId: entry.dag_run_id, + taskId: entry.task_id, + }); + + void Promise.resolve(navigate(baseUrl)); + break; + } + default: } - default: - } - }, - onHover: (_event, elements, chart) => { - chart.canvas.style.cursor = elements.length > 0 ? "pointer" : "default"; - }, - plugins: { - annotation: { - annotations: { - queuedAnnotation, - runAnnotation, - }, }, - tooltip: { - callbacks: { - label: (context) => { - const datasetLabel = context.dataset.label ?? ""; + onHover: (_event, elements, chart) => { + chart.canvas.style.cursor = elements.length > 0 ? "pointer" : "default"; + }, + plugins: { + annotation: { + annotations: { + medianAnnotation, + }, + }, + legend: { + display: true, + position: "bottom", + }, + tooltip: { + callbacks: { + label: (context) => { + const datasetLabel = context.dataset.label ?? ""; - const formatted = renderDuration(context.parsed.y, false) ?? "0"; + const formatted = renderDuration(context.parsed.y, false) ?? "0"; - return datasetLabel ? `${datasetLabel}: ${formatted}` : formatted; + return datasetLabel ? `${datasetLabel}: ${formatted}` : formatted; + }, }, }, }, - }, - responsive: true, - scales: { - x: { - stacked: true, - ticks: { - callback: (_value, index) => - formatDate(entries[index]?.run_after, selectedTimezone, getTickLabelFormat(entries)), - maxTicksLimit: 3, + responsive: true, + scales: { + x: { + stacked: true, + ticks: { + callback: (_value, index) => + formatDate(entries[index]?.run_after, selectedTimezone, getTickLabelFormat(entries)), + maxTicksLimit: 3, + }, + title: { align: "end", display: true, text: translate("common:dagRun.runAfter") }, }, - title: { align: "end", display: true, text: translate("common:dagRun.runAfter") }, - }, - y: { - ticks: { - callback: (value) => { - const num = typeof value === "number" ? value : Number(value); - - return renderDuration(num, false) ?? "0"; + y: { + beginAtZero: true, + stacked: true, + ticks: { + callback: (value) => + renderCompactDuration(typeof value === "number" ? value : Number(value)), + stepSize: getDurationTickStep(Math.max(...totalDurations, 0)), }, + title: { align: "end", display: true, text: translate("common:duration") }, }, - title: { align: "end", display: true, text: translate("common:duration") }, }, - }, - }} - /> + }} + /> + ); }; diff --git a/airflow-core/src/airflow/ui/src/pages/Dag/Overview/Overview.tsx b/airflow-core/src/airflow/ui/src/pages/Dag/Overview/Overview.tsx index d66be6299c42a..3efeec528cc3d 100644 --- a/airflow-core/src/airflow/ui/src/pages/Dag/Overview/Overview.tsx +++ b/airflow-core/src/airflow/ui/src/pages/Dag/Overview/Overview.tsx @@ -128,9 +128,17 @@ export const Overview = () => { /> - + {isLoadingRuns ? ( - + ) : ( { startDate={startDate} /> - - + + {isLoadingTaskInstances ? ( - + ) : ( )} - + ); }; diff --git a/airflow-core/src/airflow/ui/src/utils/datetimeUtils.test.ts b/airflow-core/src/airflow/ui/src/utils/datetimeUtils.test.ts index 3d381e1dc9590..551b4ca084ac0 100644 --- a/airflow-core/src/airflow/ui/src/utils/datetimeUtils.test.ts +++ b/airflow-core/src/airflow/ui/src/utils/datetimeUtils.test.ts @@ -20,7 +20,13 @@ import dayjs from "dayjs"; import dayjsDuration from "dayjs/plugin/duration"; import { describe, it, expect, vi, beforeAll, afterAll } from "vitest"; -import { getDuration, renderDuration, getRelativeTime } from "./datetimeUtils"; +import { + getDuration, + getDurationTickStep, + renderCompactDuration, + renderDuration, + getRelativeTime, +} from "./datetimeUtils"; dayjs.extend(dayjsDuration); @@ -122,3 +128,45 @@ describe("getRelativeTime", () => { expect(getRelativeTime(futureDate)).toBe("in a few seconds"); }); }); + +describe("renderCompactDuration", () => { + it.each([ + [0, "0s"], + [-5, "0s"], + [Number.NaN, "0s"], + [Number.POSITIVE_INFINITY, "0s"], + [0.25, "250ms"], + [45, "45s"], + [540, "9m"], + [545, "9m 5s"], + [3600, "1h"], + [5400, "1h 30m"], + [86_400, "1d"], + [102_600, "1d 4h"], + ])("formats %s seconds as %s", (seconds, expected) => { + expect(renderCompactDuration(seconds)).toBe(expected); + }); +}); + +describe("getDurationTickStep", () => { + it.each([ + [0, 1], + [-1, 1], + [Number.NaN, 1], + [8, 1], + [45, 10], + [300, 60], + [2000, 300], + [36_000, 7200], + ])("picks a %s second range step of %s seconds", (maxSeconds, expected) => { + expect(getDurationTickStep(maxSeconds)).toBe(expected); + }); + + it("keeps the tick count within the requested budget", () => { + expect(getDurationTickStep(2000) * 8).toBeGreaterThanOrEqual(2000); + }); + + it("falls back to an even split beyond the largest known step", () => { + expect(getDurationTickStep(10_000_000)).toBe(1_250_000); + }); +}); diff --git a/airflow-core/src/airflow/ui/src/utils/datetimeUtils.ts b/airflow-core/src/airflow/ui/src/utils/datetimeUtils.ts index 77e08fa0a8865..a86be3b415d2a 100644 --- a/airflow-core/src/airflow/ui/src/utils/datetimeUtils.ts +++ b/airflow-core/src/airflow/ui/src/utils/datetimeUtils.ts @@ -54,6 +54,56 @@ export const renderDuration = ( return duration.asSeconds() < 86_400 ? duration.format("HH:mm:ss") : duration.format("D[d]HH:mm:ss"); }; +// Chart axes need whole units at a glance; HH:mm:ss forces the reader to decode +// every tick to work out the magnitude. +export const renderCompactDuration = (durationSeconds: number): string => { + if (!Number.isFinite(durationSeconds) || durationSeconds <= 0) { + return "0s"; + } + + if (durationSeconds < 1) { + return `${Math.round(durationSeconds * 1000)}ms`; + } + + const duration = dayjs.duration(Math.round(durationSeconds), "seconds"); + const days = Math.floor(duration.asDays()); + const hours = duration.hours(); + const minutes = duration.minutes(); + const seconds = duration.seconds(); + + if (days > 0) { + return hours > 0 ? `${days}d ${hours}h` : `${days}d`; + } + + if (hours > 0) { + return minutes > 0 ? `${hours}h ${minutes}m` : `${hours}h`; + } + + if (minutes > 0) { + return seconds > 0 ? `${minutes}m ${seconds}s` : `${minutes}m`; + } + + return `${seconds}s`; +}; + +// Chart.js picks decimal steps, which on a time axis reads as 26m 40s / 33m 20s. +// Snapping to units people actually count in keeps the ticks legible. +const DURATION_TICK_STEPS_SECONDS = [ + 1, 2, 5, 10, 15, 30, 60, 120, 300, 600, 900, 1800, 3600, 7200, 10_800, 21_600, 43_200, 86_400, 172_800, + 604_800, +]; + +export const getDurationTickStep = (maxSeconds: number, maxTicks = 8): number => { + if (!Number.isFinite(maxSeconds) || maxSeconds <= 0) { + return 1; + } + + return ( + DURATION_TICK_STEPS_SECONDS.find((candidate) => maxSeconds / candidate <= maxTicks) ?? + Math.ceil(maxSeconds / maxTicks) + ); +}; + export const getDuration = ( startDate?: string | null, endDate?: string | null, diff --git a/airflow-core/src/airflow/ui/src/utils/median.test.ts b/airflow-core/src/airflow/ui/src/utils/median.test.ts new file mode 100644 index 0000000000000..6d0dc372e9a33 --- /dev/null +++ b/airflow-core/src/airflow/ui/src/utils/median.test.ts @@ -0,0 +1,51 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { describe, it, expect } from "vitest"; + +import { median } from "./median"; + +describe("median", () => { + it("returns 0 for an empty list", () => { + expect(median([])).toBe(0); + }); + + it("returns the middle value for an odd number of entries", () => { + expect(median([30, 10, 20])).toBe(20); + }); + + it("averages the two middle values for an even number of entries", () => { + expect(median([10, 20, 30, 40])).toBe(25); + }); + + it("sorts numerically rather than lexicographically", () => { + expect(median([9, 10, 100])).toBe(10); + }); + + it("does not mutate the input", () => { + const values = [30, 10, 20]; + + median(values); + + expect(values).toStrictEqual([30, 10, 20]); + }); + + it("stays near the bulk of the runs when one run is stuck", () => { + expect(median([60, 62, 58, 61, 36_000])).toBe(61); + }); +}); diff --git a/airflow-core/src/airflow/ui/src/utils/median.ts b/airflow-core/src/airflow/ui/src/utils/median.ts new file mode 100644 index 0000000000000..eb3526a3528db --- /dev/null +++ b/airflow-core/src/airflow/ui/src/utils/median.ts @@ -0,0 +1,31 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +export const median = (values: Array): number => { + if (values.length === 0) { + return 0; + } + + const sorted = [...values].sort((first, second) => first - second); + const middle = Math.floor(sorted.length / 2); + + return sorted.length % 2 === 0 + ? ((sorted[middle - 1] ?? 0) + (sorted[middle] ?? 0)) / 2 + : (sorted[middle] ?? 0); +};