From 538ca55131fc4be46a26ba0cb751391f3018ce8e Mon Sep 17 00:00:00 2001 From: Kunal Shroff Date: Fri, 3 Oct 2025 14:21:10 -0400 Subject: [PATCH 1/4] Simple impl for LTTB array down sampling function --- .../array/ArraySampleWithLTTBFunction.java | 119 ++++++++++++++++++ ...studio.apputil.formula.spi.FormulaFunction | 1 + .../ArraySampleWithLTTBFunctionTest.java | 56 +++++++++ 3 files changed, 176 insertions(+) create mode 100644 core/formula/src/main/java/org/csstudio/apputil/formula/array/ArraySampleWithLTTBFunction.java create mode 100644 core/formula/src/test/java/org/csstudio/apputil/formula/array/ArraySampleWithLTTBFunctionTest.java diff --git a/core/formula/src/main/java/org/csstudio/apputil/formula/array/ArraySampleWithLTTBFunction.java b/core/formula/src/main/java/org/csstudio/apputil/formula/array/ArraySampleWithLTTBFunction.java new file mode 100644 index 0000000000..994aab1e2e --- /dev/null +++ b/core/formula/src/main/java/org/csstudio/apputil/formula/array/ArraySampleWithLTTBFunction.java @@ -0,0 +1,119 @@ +package org.csstudio.apputil.formula.array; + +import org.epics.util.array.*; +import org.epics.vtype.*; +import org.phoebus.core.vtypes.VTypeHelper; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +/** + * @author Kunal Shroff + */ +public class ArraySampleWithLTTBFunction extends BaseArrayFunction { + + @Override + public String getName() { + return "arraySampleWithLTTB"; + } + + @Override + public String getDescription() { + return "Downsample the array using LTTB"; + } + + @Override + public List getArguments() { + return List.of("array", "buckets"); + } + + + protected VType getArrayData(final VNumberArray array, final double buckets) { + return VNumberArray.of(sampleWithLTTB(array.getData(), buckets), Alarm.none(), array.getTime(), Display.none()); + } + + // TODO at some point we might want to support non uniform x values + private static final class Point { + public final double x, y; + public Point(double x, double y) { this.x = x; this.y = y; } + } + + static ListNumber sampleWithLTTB(final ListNumber data, final double threshold) { + final int n = data.size(); + if (threshold >= n || threshold <= 0) return data; + if (threshold == 1) return CollectionNumbers.toListDouble(data.getDouble(0)); + if (threshold == 2) return CollectionNumbers.toListDouble(data.getDouble(0), data.getDouble(1)); + + final double[] out = new double[(int) threshold]; + out[0] = data.getDouble(0); // keep first + out[(int) (threshold-1)] = data.getDouble(n-1); + + int aIdx = 0; // index of last selected point + double bucketSize = (double)(n - 2) / (threshold - 2); + + for (int i = 0; i < threshold - 2; i++) { + // range of current bucket + int cs = (int)Math.floor(i * bucketSize) + 1; + int ce = (int)Math.floor((i + 1) * bucketSize) + 1; + if (ce > n - 1) ce = n - 1; + + // range of next bucket + int ns = (int)Math.floor((i + 1) * bucketSize) + 1; + int ne = (int)Math.floor((i + 2) * bucketSize) + 1; + if (ne > n - 1) ne = n - 1; + + // average of next bucket (x = index, y = value) + double avgX; + double avgY; + if (ns == ne && ns < n) { + avgX = ns; + avgY = data.getDouble(ns); + } else if (ns >= ne) { + avgX = ns; + avgY = data.getDouble(Math.min(ns, n - 2)); + } else { + double sx = 0, sy = 0; + for (int j = ns; j < ne; j++) { sx += j; sy += data.getDouble(j); } + int cnt = ne - ns; + avgX = sx / cnt; + avgY = sy / cnt; + } + + // find point in current bucket with max triangle area + double ax = aIdx, ay = data.getDouble(aIdx); + double bestArea = -1; + int bestIdx = cs; + for (int j = cs; j < ce; j++) { + double bx = j, by = data.getDouble(j); + double area = Math.abs( + (bx - ax) * (avgY - ay) - (avgX - ax) * (by - ay) + ); + if (area >= bestArea) { // prefer last index in case of tie + bestArea = area; + bestIdx = j; + } + } + + out[i+1] = data.getDouble(bestIdx); + aIdx = bestIdx; + } + + return CollectionNumbers.toList(out); + } + + @Override + public VType compute(final VType... args) throws Exception { + if (args.length != 2) { + throw new Exception("Function " + getName() + + " requires 2 arguments but received " + Arrays.toString(args)); + } + if (!VTypeHelper.isNumericArray(args[0])) + throw new Exception("Function " + getName() + + " takes array but received " + Arrays.toString(args)); + + double buckets = VTypeHelper.toDouble(args[1]); + + return getArrayData((VNumberArray) args[0], buckets); + } +} diff --git a/core/formula/src/main/resources/META-INF/services/org.csstudio.apputil.formula.spi.FormulaFunction b/core/formula/src/main/resources/META-INF/services/org.csstudio.apputil.formula.spi.FormulaFunction index 9e4092b770..8380212c2e 100644 --- a/core/formula/src/main/resources/META-INF/services/org.csstudio.apputil.formula.spi.FormulaFunction +++ b/core/formula/src/main/resources/META-INF/services/org.csstudio.apputil.formula.spi.FormulaFunction @@ -59,6 +59,7 @@ org.csstudio.apputil.formula.array.ArrayScalarDivisionFunction org.csstudio.apputil.formula.array.ArrayInverseScalarDivisionFunction org.csstudio.apputil.formula.array.ArrayOfFunction org.csstudio.apputil.formula.array.ArrayRangeOfFunction +org.csstudio.apputil.formula.array.ArraySampleWithLTTBFunction org.csstudio.apputil.formula.array.ArraySampleWithStrideFunction org.csstudio.apputil.formula.array.ArrayStatsFunction org.csstudio.apputil.formula.array.ArrayMaxFunction diff --git a/core/formula/src/test/java/org/csstudio/apputil/formula/array/ArraySampleWithLTTBFunctionTest.java b/core/formula/src/test/java/org/csstudio/apputil/formula/array/ArraySampleWithLTTBFunctionTest.java new file mode 100644 index 0000000000..d8604f697c --- /dev/null +++ b/core/formula/src/test/java/org/csstudio/apputil/formula/array/ArraySampleWithLTTBFunctionTest.java @@ -0,0 +1,56 @@ +package org.csstudio.apputil.formula.array; + +import org.epics.util.array.*; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.*; + +class ArraySampleWithLTTBFunctionTest { + @Test + void testThresholdEqualsOne() { + ListNumber data = CollectionNumbers.toListDouble(1.0, 2.0, 3.0, 4.0); + ListNumber result = ArraySampleWithLTTBFunction.sampleWithLTTB(data, 1); + assertEquals(1, result.size()); + assertEquals(1.0, result.getDouble(0)); + } + + @Test + void testThresholdEqualsTwo() { + ListNumber data = CollectionNumbers.toListDouble(1.0, 2.0, 3.0, 4.0); + ListNumber result = ArraySampleWithLTTBFunction.sampleWithLTTB(data, 2); + assertEquals(2, result.size()); + assertEquals(1.0, result.getDouble(0)); + assertEquals(2.0, result.getDouble(1)); + } + + @Test + void testThresholdGreaterThanDataSize() { + ListNumber data = CollectionNumbers.toListDouble(1.0, 2.0, 3.0); + ListNumber result = ArraySampleWithLTTBFunction.sampleWithLTTB(data, 5); + assertEquals(data.size(), result.size()); + for (int i = 0; i < data.size(); i++) { + assertEquals(data.getDouble(i), result.getDouble(i)); + } + } + + @Test + void testMonotonicIncreasing() { + ListNumber data = CollectionNumbers.toListDouble(1.0, 2.0, 3.0, 4.0, 5.0, 6.0); + ListNumber result = ArraySampleWithLTTBFunction.sampleWithLTTB(data, 3); + assertEquals(3, result.size()); + assertEquals(1.0, result.getDouble(0)); + assertEquals(5.0, result.getDouble(1)); + assertEquals(6.0, result.getDouble(2)); + } + + @Test + void testConstantData() { + ListNumber data = CollectionNumbers.toListDouble(5.0, 5.0, 5.0, 5.0, 5.0); + ListNumber result = ArraySampleWithLTTBFunction.sampleWithLTTB(data, 3); + assertEquals(3, result.size()); + assertEquals(5.0, result.getDouble(0)); + assertEquals(5.0, result.getDouble(1)); + assertEquals(5.0, result.getDouble(2)); + } + +} + From 5c42927ee6fce466e064e68ecef2decc6a27f46a Mon Sep 17 00:00:00 2001 From: Kunal Shroff Date: Fri, 3 Oct 2025 14:26:01 -0400 Subject: [PATCH 2/4] Update the array formulas documentation --- core/formula/doc/index.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/formula/doc/index.rst b/core/formula/doc/index.rst index 8e505aad0e..f4ffd3f03d 100644 --- a/core/formula/doc/index.rst +++ b/core/formula/doc/index.rst @@ -155,6 +155,10 @@ This includes the average, min, max, and element count **arrayMin(VNumberArray array)** - Returns a VDouble with the smallest value of the given array +**arraySampleWithLTTB(VNumberArray array, VNumber threshold)** - Returns a VNumberArray which is a down-sampled version of the input array. +The threshold parameter defines the maximum number of data points to return. +The down-sampling is performed using the Largest-Triangle-Three-Buckets (LTTB) algorithm. + **arraySampleWithStride(VNumberArray array, VNumber stride, VNumber offset)** - Returns a VNumberArray where each element is defined as array\[x \* stride + offset\]. **arrayCumSum(VNumberArray array)** - Returns a VNumberArray where each element is defined as the cumulative sum of the input array. From 4a86ad3113c6724f7ec4ce6bc3771d13061fdf55 Mon Sep 17 00:00:00 2001 From: Kunal Shroff Date: Mon, 6 Oct 2025 09:54:45 -0400 Subject: [PATCH 3/4] Improve the error reporting for array formulas ( make it consistent ) --- .../array/ArraySampleWithLTTBFunction.java | 15 +++++++++------ .../array/ArraySampleWithStrideFunction.java | 15 ++++++++++----- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/core/formula/src/main/java/org/csstudio/apputil/formula/array/ArraySampleWithLTTBFunction.java b/core/formula/src/main/java/org/csstudio/apputil/formula/array/ArraySampleWithLTTBFunction.java index 994aab1e2e..c08e259bc3 100644 --- a/core/formula/src/main/java/org/csstudio/apputil/formula/array/ArraySampleWithLTTBFunction.java +++ b/core/formula/src/main/java/org/csstudio/apputil/formula/array/ArraySampleWithLTTBFunction.java @@ -1,5 +1,6 @@ package org.csstudio.apputil.formula.array; +import org.csstudio.apputil.formula.Formula; import org.epics.util.array.*; import org.epics.vtype.*; import org.phoebus.core.vtypes.VTypeHelper; @@ -7,6 +8,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.logging.Level; /** * @author Kunal Shroff @@ -108,12 +110,13 @@ public VType compute(final VType... args) throws Exception { throw new Exception("Function " + getName() + " requires 2 arguments but received " + Arrays.toString(args)); } - if (!VTypeHelper.isNumericArray(args[0])) - throw new Exception("Function " + getName() + + if (!VTypeHelper.isNumericArray(args[0])) { + Formula.logger.log(Level.WARNING, "Function " + getName() + " takes array but received " + Arrays.toString(args)); - - double buckets = VTypeHelper.toDouble(args[1]); - - return getArrayData((VNumberArray) args[0], buckets); + return DEFAULT_NAN_DOUBLE_ARRAY; + } else { + double buckets = VTypeHelper.toDouble(args[1]); + return getArrayData((VNumberArray) args[0], buckets); + } } } diff --git a/core/formula/src/main/java/org/csstudio/apputil/formula/array/ArraySampleWithStrideFunction.java b/core/formula/src/main/java/org/csstudio/apputil/formula/array/ArraySampleWithStrideFunction.java index 3406c62b27..2c6cda1092 100644 --- a/core/formula/src/main/java/org/csstudio/apputil/formula/array/ArraySampleWithStrideFunction.java +++ b/core/formula/src/main/java/org/csstudio/apputil/formula/array/ArraySampleWithStrideFunction.java @@ -1,5 +1,6 @@ package org.csstudio.apputil.formula.array; +import org.csstudio.apputil.formula.Formula; import org.epics.util.array.ListDouble; import org.epics.util.array.ListNumber; import org.epics.vtype.Alarm; @@ -10,6 +11,7 @@ import java.util.Arrays; import java.util.List; +import java.util.logging.Level; /** * A formula function for extracting elements from the given array at regular intervals (stride), @@ -104,13 +106,16 @@ public VType compute(final VType... args) throws Exception { throw new Exception("Function " + getName() + " requires 2 or 3 arguments but received " + Arrays.toString(args)); } - if (!VTypeHelper.isNumericArray(args[0])) - throw new Exception("Function " + getName() + + if (!VTypeHelper.isNumericArray(args[0])) { + Formula.logger.log(Level.WARNING, "Function " + getName() + " takes array but received " + Arrays.toString(args)); + return DEFAULT_NAN_DOUBLE_ARRAY; - double stride = VTypeHelper.toDouble(args[1]); - double offset = (args.length == 3) ? VTypeHelper.toDouble(args[2]) : 0; // Default offset to 0 + } else { + double stride = VTypeHelper.toDouble(args[1]); + double offset = (args.length == 3) ? VTypeHelper.toDouble(args[2]) : 0; // Default offset to 0 - return getArrayData((VNumberArray) args[0], stride, offset); + return getArrayData((VNumberArray) args[0], stride, offset); + } } } From 0ca72a9ef9a550f280985d181d7ed7855a58c2e3 Mon Sep 17 00:00:00 2001 From: Kunal Shroff Date: Mon, 6 Oct 2025 09:57:56 -0400 Subject: [PATCH 4/4] fic imports - removing wild cards from imports --- .../formula/array/ArraySampleWithLTTBFunction.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/core/formula/src/main/java/org/csstudio/apputil/formula/array/ArraySampleWithLTTBFunction.java b/core/formula/src/main/java/org/csstudio/apputil/formula/array/ArraySampleWithLTTBFunction.java index c08e259bc3..c5eeaba0ea 100644 --- a/core/formula/src/main/java/org/csstudio/apputil/formula/array/ArraySampleWithLTTBFunction.java +++ b/core/formula/src/main/java/org/csstudio/apputil/formula/array/ArraySampleWithLTTBFunction.java @@ -1,11 +1,14 @@ package org.csstudio.apputil.formula.array; import org.csstudio.apputil.formula.Formula; -import org.epics.util.array.*; -import org.epics.vtype.*; +import org.epics.util.array.CollectionNumbers; +import org.epics.util.array.ListNumber; +import org.epics.vtype.Alarm; +import org.epics.vtype.Display; +import org.epics.vtype.VNumberArray; +import org.epics.vtype.VType; import org.phoebus.core.vtypes.VTypeHelper; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.logging.Level;