diff --git a/.gitignore b/.gitignore index 19ffc25..54414ac 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,12 @@ tests/.env *.orig *.log *.pot + +# Local test/demo databases (created by examples and tests) +*.sqlite +*.sqlite-journal +*.sqlite-wal +*.sqlite-shm __pycache__/* .cache/* .*.swp diff --git a/README.md b/README.md index b3c6f99..872edc4 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,23 @@ See [examples/datatool_dashboard.py](examples/datatool_dashboard.py) for a full To regenerate the screenshots after UI changes, see [docs/generate_screenshots.py](docs/generate_screenshots.py). +### Export + +A toolbar at the top of the plot card exports the currently plotted data and +plot (requires the `export` extra): + +- **Download CSV / XLSX** - a unit-aware [pint-pandas](https://github.com/hgrecco/pint-pandas) + table (one column per series, its unit written in a dedicated header row via + `dequantify()`), matching exactly what is plotted (unit conversion + + normalization). Rows are capped at `EXPORT_MAX_ROWS` (default 1,000,000) via + uniform downsampling. +- **Download plot (HTML)** - a standalone interactive Bokeh document + (`bokeh.embed.file_html`). PNG is available from the Bokeh toolbar's save tool. + +The rendered figures are exposed via `view.figures` so a host app can add its +own annotations; `view.export_series()` returns the plotted series as tidy +records. + ## Server-side downsampling Large time series are downsampled on the server so the plot only transports the @@ -332,6 +349,21 @@ regenerate these screenshots. pip install opensemantic.base # models only pip install opensemantic.base[controller] # + aiosqlite, postgrest pip install opensemantic.base[view] # + panel, bokeh, panelini, pint +pip install opensemantic.base[export] # + pandas, pint-pandas, openpyxl (data/plot export) +``` + +### Running the examples + +```bash +git clone https://github.com/OpenSemanticWorld-Packages/opensemantic.base-python.git +cd opensemantic.base-python +pip install -e . +pip install opensemantic.base[controller] opensemantic.base[view] opensemantic.base[export] +pip install opensemantic.characteristics.quantitative # typed characteristics +pip install nest-asyncio # examples/datatool_dashboard.py +pip install PyJWT # examples/downsample_demo.py (needs a database) + +panel serve examples/datatool_dashboard.py --dev ``` ## Testing diff --git a/setup.cfg b/setup.cfg index 1631fcb..7ad953b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -71,6 +71,12 @@ view = panelini pint +# Data/plot export from the dashboard views (unit-aware CSV/XLSX + plot HTML). +export = + pandas + pint-pandas + openpyxl + # Add here test requirements (semicolon/line-separated) testing = setuptools diff --git a/src/opensemantic/base/view/_base_view.py b/src/opensemantic/base/view/_base_view.py index 9cf1736..5add5f6 100644 --- a/src/opensemantic/base/view/_base_view.py +++ b/src/opensemantic/base/view/_base_view.py @@ -16,6 +16,7 @@ """ import asyncio +import io import logging from typing import Any, List, Tuple @@ -37,6 +38,108 @@ COLORS = Category10_10 +# Row cap for data export (uniform np.linspace downsampling above this). +EXPORT_MAX_ROWS = 1_000_000 + +# Optional dependencies for unit-aware data export (opensemantic.base[export]). +try: + import pandas as _pd # noqa: F401 + import pint_pandas as _pint_pandas # noqa: F401 + + _EXPORT_DEPS_OK = True +except Exception: # pragma: no cover - optional + _EXPORT_DEPS_OK = False + + +def _series_to_dataframe(series: List[dict], max_rows: int): + """Tidy series records -> a DataFrame (one column per series). + + Each record ``{"label", "x", "y", "x_kind", "unit"}`` becomes a column, + aligned (outer join) on the x index. Numeric records with a unit get a + ``pint[]`` dtype; a record with no unit (e.g. a text-log channel) or + non-numeric values becomes a plain object column, so all checked channels - + not just the plotted numeric ones - are exportable. Rows are capped at + ``max_rows`` via uniform np.linspace downsampling (logged). Returns ``None`` + if pandas/pint-pandas are missing or there is nothing to export. + """ + try: + import numpy as np + import pandas as pd + import pint_pandas # noqa: F401 + except ImportError: + _logger.warning( + "Data export needs the 'export' extra (pandas, pint-pandas, openpyxl)." + ) + return None + if not series: + return None + + parts = [] + seen = {} + for s in series: + idx = pd.Index(s["x"], name=s.get("x_kind") or "x") + unit = s.get("unit") + col = None + if unit: + try: + col = pd.Series(s["y"], index=idx, dtype=f"pint[{unit}]") + except Exception: + col = None + if col is None: + # No unit (text/log) or an unparseable one: numerics fall back to a + # dimensionless pint column, text values to a plain object column. + try: + col = pd.Series(s["y"], index=idx, dtype="pint[dimensionless]") + except Exception: + col = pd.Series(s["y"], index=idx, dtype="object") + # A well-defined outer join needs a unique index per column. + col = col[~col.index.duplicated(keep="last")] + # Unique column names: pandas returns a DataFrame (not a Series) for a + # duplicated label, which would break the per-column dequantify. + label = s["label"] + if label in seen: + seen[label] += 1 + label = f"{label} ({seen[label]})" + else: + seen[label] = 0 + col.name = label + parts.append(col) + if not parts: + return None + + df = pd.concat(parts, axis=1).sort_index() + n = len(df) + if n > max_rows: + sel = np.linspace(0, n - 1, max_rows).astype(int) + df = df.iloc[sel] + _logger.info("Data export capped from %d to %d rows.", n, max_rows) + return df + + +def _dequantify(df): + """Like ``df.pint.dequantify()`` but tolerant of non-pint (text) columns. + + Produces a two-level column header ``(label, unit)`` so a dedicated unit + header row is written on export and the column keys stay unit-free. Pint + columns contribute their magnitude and unit; plain (text) columns keep their + values under an empty unit. + """ + import pandas as pd + + tuples = [] + mags = [] + for name in df.columns: + s = df[name] + if str(s.dtype).startswith("pint"): + tuples.append((str(name), str(s.pint.units))) + mags.append(s.pint.magnitude) + else: + tuples.append((str(name), "")) + mags.append(s) + out = pd.concat(mags, axis=1) + out.columns = pd.MultiIndex.from_tuples(tuples, names=[None, "unit"]) + return out + class BaseDataView: """Mixin with the UI/plot pieces shared by the archive views.""" @@ -92,6 +195,190 @@ def _build_plot(self): sizing_mode="stretch_width", ) + # -- Export (data + plot) -- + + _figures: List[Any] = [] + + @property + def figures(self) -> List[Any]: + """The Bokeh figures currently rendered (updated by _build_figure). + + Exposed so host apps can add their own annotations; also used by the + HTML plot export. Empty when nothing is plotted. + """ + return list(getattr(self, "_figures", []) or []) + + def _build_export_toolbar(self): + """Compact collapsed "Export" card of FileDownload buttons. + + A dashboard-level action (data CSV/XLSX + plot HTML), so it lives in the + Plot Controls sidebar rather than inside the Time Series group. Returns + the card and stores it on ``self._export_box`` for host embedding. + """ + deps_note = "" if _EXPORT_DEPS_OK else " (needs opensemantic.base[export])" + self._download_csv = pn.widgets.FileDownload( + label=_t("download_csv", self.lang) + deps_note, + filename="dashboard_data.csv", + callback=lambda: self._build_data_export("csv"), + button_type="default", + disabled=True, + sizing_mode="stretch_width", + ) + self._download_xlsx = pn.widgets.FileDownload( + label=_t("download_xlsx", self.lang) + deps_note, + filename="dashboard_data.xlsx", + callback=lambda: self._build_data_export("xlsx"), + button_type="default", + disabled=True, + sizing_mode="stretch_width", + ) + self._download_html = pn.widgets.FileDownload( + label=_t("download_plot", self.lang), + filename="dashboard_plot.html", + callback=self._build_plot_html, + button_type="default", + disabled=True, + sizing_mode="stretch_width", + ) + self._export_box = pn.Card( + pn.Column( + self._download_csv, + self._download_xlsx, + self._download_html, + sizing_mode="stretch_width", + ), + title=_t("export", self.lang), + collapsed=True, + sizing_mode="stretch_width", + # Separate this tile from the controls above and the next section. + margin=(10, 5), + ) + return self._export_box + + def _has_export_log(self) -> bool: + """Whether the log console currently has content to include in HTML.""" + pane = getattr(self, "_log_pane", None) + card = getattr(self, "_log_card", None) + return bool(getattr(pane, "object", "")) and bool( + getattr(card, "visible", False) + ) + + def _update_export_state(self): + """Enable/disable export buttons based on what is available. + + Data (CSV/XLSX) covers every checked channel, so it keys off + ``export_series`` (numeric + text). The plot HTML covers the rendered + figures and the log console, so it keys off either being present. + """ + has_data = bool(self.export_series()) + has_html = bool(self.figures) or self._has_export_log() + for w in ("_download_csv", "_download_xlsx"): + btn = getattr(self, w, None) + if btn is not None: + btn.disabled = not (has_data and _EXPORT_DEPS_OK) + if getattr(self, "_download_html", None) is not None: + self._download_html.disabled = not has_html + + def _build_data_export(self, fmt: str) -> io.BytesIO: + """Build a unit-aware CSV/XLSX of the plotted series via pint-pandas. + + Serialized with the pint-pandas ``dequantify`` accessor so a dedicated + unit header row is written and the column keys stay unit-free. Returns a + (possibly empty) BytesIO. + """ + buf = io.BytesIO() + cap = int(getattr(self, "EXPORT_MAX_ROWS", EXPORT_MAX_ROWS)) + df = _series_to_dataframe(self.export_series(), cap) + if df is None or df.empty: + return buf + import pandas as pd + + dequantified = _dequantify(df) + if fmt == "xlsx": + try: + with pd.ExcelWriter(buf, engine="openpyxl") as writer: + dequantified.to_excel(writer) + except ImportError: + _logger.warning("XLSX export needs openpyxl (base[export]).") + return io.BytesIO() + else: + buf.write(dequantified.to_csv().encode("utf-8")) + buf.seek(0) + return buf + + def _export_figures(self) -> List[Any]: + """Figures to serialize for HTML export. + + Defaults to the live ``figures``; views whose figures are attached to a + Panel document override this to return a fresh, unattached copy, since + ``file_html`` requires models that belong to no other document. + """ + return self.figures + + #: Fixed frame size for HTML-exported figures. + EXPORT_PLOT_WIDTH = 1000 + EXPORT_PLOT_HEIGHT = 350 + + def _export_extra_models(self) -> List[Any]: + """Extra Bokeh models to append below the figures in the HTML export. + + Includes the log console (as an HTML ``Div``) when it has content, so a + dashboard with text-log channels carries them into the export too. + """ + models: List[Any] = [] + if self._has_export_log(): + from bokeh.models import Div + + # Scrollable inner container so a long log does not stretch the page; + # the heading stays fixed above it. + width = self.EXPORT_PLOT_WIDTH + models.append( + Div( + text=( + f"

{_t('log_console', self.lang)}" + f"

" + # Explicit width on the box: Bokeh's Div content wrapper + # is content-sized, so match the fixed plot width here. + f"
" + f"{self._log_pane.object}
" + ), + width=width, + sizing_mode="fixed", + ) + ) + return models + + def _build_plot_html(self) -> io.BytesIO: + """Standalone interactive HTML of the figures + log console.""" + buf = io.BytesIO() + figs = self._export_figures() + extras = self._export_extra_models() + if not figs and not extras: + return buf + from bokeh.embed import file_html + from bokeh.layouts import column as bk_column + from bokeh.resources import CDN + + # The live figures are stretch_width, but a standalone HTML has no + # container width to stretch into, so the plot frame collapses to zero + # width. Pin an explicit size on the export copies so they render. + for fig in figs: + try: + fig.sizing_mode = "fixed" + fig.width = self.EXPORT_PLOT_WIDTH + fig.height = self.EXPORT_PLOT_HEIGHT + except Exception: # pragma: no cover - non-figure layout + pass + + roots = list(figs) + list(extras) + root = roots[0] if len(roots) == 1 else bk_column(*roots) + html = file_html(root, CDN, getattr(self, "_title", "Plot")) + buf.write(html.encode("utf-8")) + buf.seek(0) + return buf + def _build_log_console(self): self._log_pane = pn.pane.HTML( "", @@ -135,6 +422,7 @@ def _refresh_plot(self): """Rebuild plot and log console from the subclass's loaded data.""" self._build_figure() self._update_log_console() + self._update_export_state() def _numeric(self, value: Any, channel: Any, target_unit_name: Any) -> Any: """Convert a value to its display unit and return the numeric scalar. @@ -155,6 +443,26 @@ def _numeric(self, value: Any, channel: Any, target_unit_name: Any) -> Any: return value.get("value") return value + def _pint_unit(self, value: Any, target_unit_name: Any) -> str: + """Pint-parseable unit string for a typed value in its display unit. + + Converts the value to ``target_unit_name`` (like the plot) and reads the + pint unit off ``to_pint()``. Falls back to ``"dimensionless"`` for raw / + untyped values so the export column is always a valid pint dtype. + """ + if value is None or not hasattr(value, "to_pint"): + return "dimensionless" + try: + if target_unit_name and hasattr(value, "to_unit"): + unit = getattr(value, "unit", None) + enum = type(unit) if unit is not None else None + if enum is not None and hasattr(enum, "__members__"): + if target_unit_name in enum.__members__: + value = value.to_unit(enum[target_unit_name]) + return str(value.to_pint().units) + except Exception: + return "dimensionless" + def _get_axis_label(self, group_key: str) -> str: """Build y-axis label: characteristic name [unit symbol].""" channels = self._groups.get(group_key, []) @@ -211,6 +519,16 @@ def panel(self): def _build_figure(self): # pragma: no cover - overridden raise NotImplementedError + def export_series(self) -> List[dict]: # pragma: no cover - overridden + """Return the currently plotted series as tidy records. + + Each record: ``{"label", "x", "y", "x_kind": "datetime"|"seconds", + "unit"}`` where ``unit`` is a pint-parseable string. Built by reusing + the same trace extraction as ``_build_figure`` so the export matches + exactly what is plotted (unit conversion + normalization). + """ + raise NotImplementedError + def _update_log_console(self): # pragma: no cover - overridden raise NotImplementedError diff --git a/src/opensemantic/base/view/_channel_utils.py b/src/opensemantic/base/view/_channel_utils.py index 4f93d48..8d01d6a 100644 --- a/src/opensemantic/base/view/_channel_utils.py +++ b/src/opensemantic/base/view/_channel_utils.py @@ -621,6 +621,11 @@ def get_selected_channels( "live": {"en": "Live", "de": "Live"}, "clear_cache": {"en": "Clear Cache", "de": "Cache leeren"}, "load_range": {"en": "Load current range", "de": "Aktuellen Bereich laden"}, + "export": {"en": "Export", "de": "Export"}, + "download_csv": {"en": "Download CSV", "de": "CSV herunterladen"}, + "download_xlsx": {"en": "Download XLSX", "de": "XLSX herunterladen"}, + "download_plot": {"en": "Download plot (HTML)", "de": "Plot (HTML) herunterladen"}, + "no_data": {"en": "No data", "de": "Keine Daten"}, } diff --git a/src/opensemantic/base/view/_datatool_dashboard.py b/src/opensemantic/base/view/_datatool_dashboard.py index 12f8eac..3a0e68b 100644 --- a/src/opensemantic/base/view/_datatool_dashboard.py +++ b/src/opensemantic/base/view/_datatool_dashboard.py @@ -252,6 +252,7 @@ def _build_controls(self): self._row_limit_input, self._clear_cache_button, self._unit_controls, + self._build_export_toolbar(), title=_t("plot_controls", self.lang), ) @@ -412,10 +413,13 @@ def _downsample_for(self, channel): method = resolve_downsample_method(channel, ds.method.value) return ds.max_points, method, ds.edge_anchors - def _build_figure(self): - """Build Bokeh figures - one per characteristic group.""" - self._plot_col.clear() + def _make_figures(self): + """Build a fresh list of Bokeh figures (one per group) from the cache. + Returns ``(figs, shared_x_range)``. The figures are not attached to any + pane/document, so this is reused both for live rendering and for a + detached copy for HTML export (a model may live in only one document). + """ plot_groups = [] for group_key, channels in self._groups.items(): if not channels: @@ -427,7 +431,7 @@ def _build_figure(self): plot_groups.append((group_key, channels, vtype)) if not plot_groups: - return + return [], None # One shared x-range for every figure, so panning, zooming or resetting # any plot moves all of them together (the y-ranges stay independent). @@ -481,6 +485,15 @@ def _build_figure(self): fig.legend.click_policy = "hide" figs.append(fig) + return figs, shared_x + + def _build_figure(self): + """Build Bokeh figures - one per characteristic group.""" + self._plot_col.clear() + figs, shared_x = self._make_figures() + self._figures = figs + if not figs: + return # Keep the shared range for "Load current range" and bridge the Reset # event: figure.on_event(Reset) does not propagate through Panel's Bokeh @@ -499,6 +512,11 @@ def _build_figure(self): for fig in figs: self._plot_col.append(pn.pane.Bokeh(fig, sizing_mode="stretch_width")) + def _export_figures(self): + """Fresh, unattached figures for HTML export (see _make_figures).""" + figs, _ = self._make_figures() + return figs + def _current_xrange_window(self): """Return (start, end) *naive* datetimes of the current plot x-range. @@ -638,6 +656,73 @@ def _extract_composite_field( return timestamps, values + # -- Export -- + + def _representative_value(self, ch: Any) -> Any: + """A typed (Characteristic) value from the channel's cache, or None.""" + if ch.uuid in self._composite_parents: + parent_ch, field_name = self._composite_parents[ch.uuid] + for pt in self._cached_data.get(parent_ch.uuid, []): + val = pt.value + sub = ( + val.get(field_name) + if isinstance(val, dict) + else getattr(val, field_name, None) + ) + if sub is not None and hasattr(sub, "to_pint"): + return sub + return None + for pt in self._cached_data.get(ch.uuid, []): + if hasattr(pt.value, "to_pint"): + return pt.value + return None + + def _series_unit(self, ch: Any, group_key: str) -> str: + """Pint-parseable unit string of the plotted (display-unit) values.""" + return self._pint_unit( + self._representative_value(ch), self._unit_selections.get(group_key) + ) + + def export_series(self) -> List[dict]: + """Every checked channel as tidy records (datetime x). + + Numeric channels carry their display unit and match _build_figure's + traces; text-log channels (a different display group) are included too, + with ``unit=None`` so they export as plain text columns. + """ + records: List[dict] = [] + for group_key, channels in self._groups.items(): + if not channels: + continue + is_text = resolve_value_type(channels[0][1]) == "text" + for ctrl, ch in channels: + x, y = self._extract_trace_data(ch, group_key) + if not x: + continue + # Composite sub-fields share the parent channel label, so add + # the sub-field name to keep one column per series. + if ch.uuid in self._composite_parents: + parent_ch, field_name = self._composite_parents[ch.uuid] + label = ( + f"{get_display_label(ctrl, self.lang)}/" + f"{get_display_label(parent_ch, self.lang)}/{field_name}" + ) + else: + label = ( + f"{get_display_label(ctrl, self.lang)}/" + f"{get_display_label(ch, self.lang)}" + ) + records.append( + { + "label": label, + "x": x, + "y": y, + "x_kind": "datetime", + "unit": None if is_text else self._series_unit(ch, group_key), + } + ) + return records + def _update_log_console(self): """Update the log console with text-type channel data.""" log_entries = [] diff --git a/src/opensemantic/base/view/_process_dashboard.py b/src/opensemantic/base/view/_process_dashboard.py index 0364041..0ad574c 100644 --- a/src/opensemantic/base/view/_process_dashboard.py +++ b/src/opensemantic/base/view/_process_dashboard.py @@ -260,6 +260,7 @@ def _build_controls(self): self._row_limit_input, self._clear_cache_button, self._unit_controls, + self._build_export_toolbar(), title=_t("plot_controls", self.lang), ) @@ -360,9 +361,13 @@ async def _load_and_plot(self): self._refresh_plot() - def _build_figure(self): - self._plot_col.clear() + def _make_figures(self) -> List[Any]: + """Build a fresh list of Bokeh figures (one per group) from the traces. + The figures are not attached to any pane/document, so this is reused + both for live rendering and for a detached copy for HTML export (a model + may live in only one document). + """ # Group traces by y-axis group, skipping text channels. plot_groups: Dict[str, List[Dict[str, Any]]] = {} for tr in self._traces: @@ -375,8 +380,9 @@ def _build_figure(self): plot_groups.setdefault(gkey, []).append(tr) if not plot_groups: - return + return [] + figs: List[Any] = [] color_idx = 0 for gkey, traces in plot_groups.items(): axis_label = self._get_axis_label(gkey) @@ -395,11 +401,7 @@ def _build_figure(self): # of the same type). Label each line distinctly by # object · process / datatool / channel so Bokeh keeps them as # separate, individually-toggleable legend entries. - label = ( - f"{tr['object_label']} · {tr['process_label']} / " - f"{get_display_label(tr['controller'], self.lang)} / " - f"{get_display_label(tr['channel'], self.lang)}" - ) + label = self._trace_label(tr) src = ColumnDataSource(data={"x": xs, "y": ys}) fig.line( "x", @@ -412,8 +414,54 @@ def _build_figure(self): color_idx += 1 fig.legend.click_policy = "hide" fig.legend.label_text_font_size = "8pt" + figs.append(fig) + return figs + + def _build_figure(self): + self._plot_col.clear() + self._figures = self._make_figures() + for fig in self._figures: self._plot_col.append(pn.pane.Bokeh(fig, sizing_mode="stretch_width")) + def _export_figures(self) -> List[Any]: + """Fresh, unattached figures for HTML export (see _make_figures).""" + return self._make_figures() + + def _trace_label(self, tr: Dict[str, Any]) -> str: + return ( + f"{tr['object_label']} · {tr['process_label']} / " + f"{get_display_label(tr['controller'], self.lang)} / " + f"{get_display_label(tr['channel'], self.lang)}" + ) + + def export_series(self) -> List[Dict[str, Any]]: + """Plotted traces as tidy records (relative-seconds x).""" + records: List[Dict[str, Any]] = [] + for tr in self._traces: + ch = tr["channel"] + if resolve_value_type(ch) == "text": + continue + gkey = self._group_of.get(ch.uuid) + if gkey is None: + continue + xs, ys = self._extract_trace(tr, gkey) + if not xs: + continue + rep = next( + (p.value for p in tr["points"] if hasattr(p.value, "to_pint")), + None, + ) + records.append( + { + "label": self._trace_label(tr), + "x": xs, + "y": ys, + "x_kind": "seconds", + "unit": self._pint_unit(rep, self._unit_selections.get(gkey)), + } + ) + return records + def _extract_trace(self, tr: Dict[str, Any], group_key: str) -> Tuple[List, List]: """Relative-seconds x and (unit-converted) numeric y for a trace.""" points = tr["points"] diff --git a/tests/test_export.py b/tests/test_export.py new file mode 100644 index 0000000..9de8003 --- /dev/null +++ b/tests/test_export.py @@ -0,0 +1,221 @@ +"""Tests for the archive-view data/plot export.""" + +import asyncio +import datetime as dt +from uuid import NAMESPACE_URL, uuid5 + +import pytest + +# The export machinery lives on BaseDataView, which imports panel/panelini. +pytest.importorskip("panel") +pytest.importorskip("panelini") +pytest.importorskip("pandas") +pytest.importorskip("pint_pandas") + +from opensemantic import compute_scoped_uuid # noqa: E402 +from opensemantic.base.v1 import ( # noqa: E402 + Database, + DataChannel, + DataTool, + DataToolController, +) +from opensemantic.base.view import DataToolView # noqa: E402 +from opensemantic.base.view._base_view import ( # noqa: E402 + BaseDataView, + _series_to_dataframe, +) +from opensemantic.base.view._config import DashboardConfig, PlotConfig # noqa: E402 +from opensemantic.characteristics.quantitative.v1 import ( # noqa: E402 + Temperature, + TemperatureUnit, +) +from opensemantic.core.v1 import Label # noqa: E402 + + +class _Stub(BaseDataView): + """Minimal BaseDataView to exercise the pure export path.""" + + EXPORT_MAX_ROWS = 1_000_000 + + def __init__(self, series): + self._series = series + + def export_series(self): + return self._series + + +def _records(): + t0 = dt.datetime(2024, 1, 1, 0, 0, 0) + return [ + { + "label": "tool/temp", + "x": [t0, t0 + dt.timedelta(seconds=1)], + "y": [300.0, 301.0], + "x_kind": "datetime", + "unit": "kelvin", + }, + { + "label": "tool/volt", + "x": [t0], + "y": [1.5], + "x_kind": "datetime", + "unit": "volt", + }, + ] + + +def test_series_to_dataframe_units_and_alignment(): + df = _series_to_dataframe(_records(), 1_000_000) + assert list(df.columns) == ["tool/temp", "tool/volt"] + assert df.shape[0] == 2 # outer join over the two timestamps + assert str(df["tool/temp"].pint.units) == "kelvin" + assert str(df["tool/volt"].pint.units) == "volt" + + +def test_build_data_export_csv_has_unit_header(): + text = _Stub(_records())._build_data_export("csv").getvalue().decode() + # dequantify() writes a unit header row and keeps column keys unit-free. + assert "kelvin" in text and "volt" in text + assert "tool/temp" in text and "tool/volt" in text + + +def test_build_data_export_row_cap(): + n = 100 + t0 = dt.datetime(2024, 1, 1) + series = [ + { + "label": "A", + "x": [t0 + dt.timedelta(seconds=i) for i in range(n)], + "y": [float(i) for i in range(n)], + "x_kind": "datetime", + "unit": "kelvin", + } + ] + assert len(_series_to_dataframe(series, 10)) == 10 + + +def test_empty_series_exports_empty(): + assert _Stub([])._build_data_export("csv").getvalue() == b"" + assert _series_to_dataframe([], 10) is None + + +def test_duplicate_labels_are_disambiguated(): + # Composite sub-fields can share a label; each must stay its own column so + # dequantify sees Series (not a DataFrame) per column. + t0 = dt.datetime(2024, 1, 1) + series = [ + {"label": "A/AQ", "x": [t0], "y": [1.0], "x_kind": "datetime", "unit": "K"}, + {"label": "A/AQ", "x": [t0], "y": [2.0], "x_kind": "datetime", "unit": "volt"}, + ] + df = _series_to_dataframe(series, 1_000_000) + assert list(df.columns) == ["A/AQ", "A/AQ (1)"] + text = _Stub(series)._build_data_export("csv").getvalue().decode() + assert "kelvin" in text and "volt" in text + + +def test_text_channel_exports_as_object_column(): + # A checked text-log channel (unit=None) exports alongside numeric ones. + t0 = dt.datetime(2024, 1, 1) + series = [ + { + "label": "tool/temp", + "x": [t0], + "y": [300.0], + "x_kind": "datetime", + "unit": "kelvin", + }, + { + "label": "tool/status", + "x": [t0], + "y": ["OK"], + "x_kind": "datetime", + "unit": None, + }, + ] + df = _series_to_dataframe(series, 1_000_000) + assert list(df.columns) == ["tool/temp", "tool/status"] + assert str(df["tool/temp"].pint.units) == "kelvin" + assert df["tool/status"].dtype == object + text = _Stub(series)._build_data_export("csv").getvalue().decode() + assert "tool/status" in text and "OK" in text and "kelvin" in text + + +# -- View integration: export_series / figures / plot HTML -- + + +def _loaded_view(): + parent = uuid5(NAMESPACE_URL, "ExportSensor") + tool = DataTool( + uuid=parent, + name="ExportSensor", + label=[Label(text="Export Sensor")], + data_channels=[ + DataChannel( + uuid=str(compute_scoped_uuid(parent, "temp")), + osw_id="placeholder", + name="temperature", + label=[Label(text="Temperature")], + characteristic=Temperature.get_cls_iri(), + ), + ], + storage_locations=[Database(name="export_test_db", label=[Label(text="DB")])], + ) + ctrl = DataToolController(tool, auto_archive=True) + base = dt.datetime(2024, 1, 1, tzinfo=dt.timezone.utc) + + async def store(): + for i in range(5): + await ctrl.store_channel_data( + DataToolController.StoreChannelDataParams( + channel="temperature", + value=Temperature(value=300.0 + i, unit=TemperatureUnit.kelvin), + timestamp=base + dt.timedelta(seconds=i), + ) + ) + + asyncio.run(store()) + + view = DataToolView( + controllers=[ctrl], + config=DashboardConfig(plot=PlotConfig(auto_fetch=False)), + title="Export Test", + embeddable=True, + ) + view.set_time_range( + base - dt.timedelta(seconds=1), base + dt.timedelta(seconds=10), fetch=False + ) + for root in view._tree.source: + for child in root.get("children", []): + child["selected"] = True + view._update_selection() + view._update_unit_controls() + asyncio.run(view._load_and_plot()) + return view, ctrl + + +def test_datatool_export_series_and_figures(): + view, ctrl = _loaded_view() + try: + records = view.export_series() + assert len(records) == 1 + rec = records[0] + assert set(rec) == {"label", "x", "y", "x_kind", "unit"} + assert rec["x_kind"] == "datetime" + assert rec["unit"] == "kelvin" + assert rec["y"] == [300.0, 301.0, 302.0, 303.0, 304.0] + assert len(view.figures) == 1 + html = view._build_plot_html().getvalue().decode() + assert " 0) return el.shadowRoot; + const deeper = findWbShadow(el.shadowRoot); + if (deeper) return deeper; + } + } + return null; +} +""" + + +def _click_checkbox(page, idx): + """Click the idx-th Wunderbaum checkbox (piercing the shadow root).""" + page.evaluate( + f"""() => {{ + {_WB_SHADOW_JS} + const wbRoot = findWbShadow(document); + if (wbRoot) {{ + const cbs = wbRoot.querySelectorAll('i.wb-checkbox'); + if (cbs[{idx}]) cbs[{idx}].click(); + }} + }}""" + ) + + +def _download(page, label): + """Click a FileDownload button and return the downloaded file path. + + Uses the button role (Playwright pierces Bokeh's shadow DOM); the download + is generated lazily server-side on click. + """ + btn = page.get_by_role("button", name=label) + btn.scroll_into_view_if_needed() + with page.expect_download(timeout=20000) as dl: + btn.click() + return dl.value.path() + + +def _wait_ready(timeout=40): + # A raw TCP connect (not an HTTP GET) avoids corporate-proxy routing that + # would otherwise divert a localhost request. The app is imported before + # the port opens, so an accepting socket means the dashboard is ready. + deadline = time.time() + timeout + while time.time() < deadline: + try: + with socket.create_connection(("127.0.0.1", PORT), timeout=2): + return True + except OSError: + time.sleep(1) + return False + + +def _kill_port(port): + """Free a TCP port on Windows (leftover panel servers block re-runs). + + Parses ``netstat`` by columns rather than the state word, which is + localized (e.g. German "ABHOEREN" instead of "LISTENING"). + """ + if os.name != "nt": + return + try: + out = subprocess.run( + ["netstat", "-ano"], capture_output=True, text=True, timeout=10 + ).stdout + except Exception: + return + pids = set() + for line in out.splitlines(): + parts = line.split() + # proto local foreign state pid -> a listening socket's local address + # ends with :port and its pid is non-zero. + if len(parts) >= 5 and parts[1].endswith(f":{port}") and parts[-1] != "0": + pids.add(parts[-1]) + for pid in pids: + subprocess.run(["taskkill", "/F", "/T", "/PID", pid], capture_output=True) + + +def _terminate(proc): + """Kill the whole panel server process tree (Windows-safe).""" + if os.name == "nt": + subprocess.run( + ["taskkill", "/F", "/T", "/PID", str(proc.pid)], + capture_output=True, + ) + else: + proc.terminate() + try: + proc.wait(timeout=8) + except subprocess.TimeoutExpired: + proc.kill() + + +@pytest.fixture(scope="module") +def server(): + # Serve an isolated copy of the example: its SQLite DB path is + # ``Path(__file__).parent``, so a copy in a fresh temp dir gets its own DB + # and never contends with a running demo on the shared example file. + _kill_port(PORT) # clear any leftover server from an aborted run + workdir = tempfile.mkdtemp(prefix="export_browser_") + # Keep the filename so the served route stays /datatool_dashboard. + app = os.path.join(workdir, "datatool_dashboard.py") + shutil.copyfile(EXAMPLE, app) + # Log to a file (not PIPE): a full PIPE buffer would stall the child during + # the verbose startup and the server would never come up. + log = tempfile.NamedTemporaryFile( + prefix="panel_export_test_", suffix=".log", delete=False + ) + proc = subprocess.Popen( + [ + sys.executable, + "-m", + "panel", + "serve", + app, + "--port", + str(PORT), + "--allow-websocket-origin", + f"localhost:{PORT}", + ], + stdout=log, + stderr=subprocess.STDOUT, + cwd=workdir, + ) + try: + if not _wait_ready(): + proc.terminate() + log.flush() + with open(log.name, "r", encoding="utf-8", errors="replace") as fh: + out = fh.read() + pytest.skip(f"panel serve did not become ready:\n{out[-1200:]}") + yield URL + finally: + _terminate(proc) + log.close() + try: + os.remove(log.name) + except OSError: + pass + shutil.rmtree(workdir, ignore_errors=True) + + +@pytest.fixture(scope="module") +def page(server): + try: + with sync_playwright() as p: + try: + browser = p.chromium.launch(headless=True) + except Exception as e: + pytest.skip(f"Chromium not available: {e}") + context = browser.new_context( + viewport={"width": 1400, "height": 900}, accept_downloads=True + ) + pg = context.new_page() + pg.goto(server, timeout=30000) + pg.wait_for_timeout(6000) + # Select "Sensor A" parent (checkbox 0) -> all its channels, incl. + # the text "Status" channel. auto_fetch is on, so data loads. + _click_checkbox(pg, 0) + pg.wait_for_timeout(5000) + # Expand the collapsed "Export" card in Plot Controls. + header = pg.get_by_text("Export", exact=True).first + header.scroll_into_view_if_needed() + header.click() + pg.wait_for_timeout(1500) + yield pg + browser.close() + except Exception as e: # pragma: no cover - environment guard + pytest.skip(f"Playwright session failed: {e}") + + +def test_download_csv_has_units_and_text_channel(page): + path = _download(page, "Download CSV") + with open(path, "r", encoding="utf-8") as fh: + text = fh.read() + # Numeric channel with a unit header row, plus the text "Status" channel. + assert "Temperature" in text + assert "unit" in text + assert "Status" in text + # More than just the two header rows (real data present). + assert len(text.splitlines()) > 3 + + +# Recurse through open shadow roots (Bokeh 3 renders its canvas in one). +_CANVAS_W_JS = """ +() => { + let maxw = 0; + const walk = (root) => { + for (const c of root.querySelectorAll('canvas')) { + maxw = Math.max(maxw, Math.round(c.getBoundingClientRect().width)); + } + for (const el of root.querySelectorAll('*')) { + if (el.shadowRoot) walk(el.shadowRoot); + } + }; + walk(document); + return maxw; +} +""" + + +def test_download_plot_html(page): + path = _download(page, "Download plot (HTML)") + with open(path, "rb") as fh: + low = fh.read().lower() + assert b" 200 + finally: + viewer.close() diff --git a/tox.ini b/tox.ini index f7025ab..205c0d5 100644 --- a/tox.ini +++ b/tox.ini @@ -20,6 +20,7 @@ extras = testing controller view + export deps = pyjwt python-dotenv