From b3e20cd9def5168ab3adfc5620cf9249d330ab08 Mon Sep 17 00:00:00 2001 From: Natalia Borovskikh Date: Tue, 3 Mar 2026 13:05:00 +0000 Subject: [PATCH 01/24] Add very basic arcadia->urdb conversion --- pyproject.toml | 4 +- tariff_fetch/_cli/genability.py | 5 +- tariff_fetch/arcadia/api.py | 2 + tariff_fetch/arcadia/schema/tariffproperty.py | 6 +- tariff_fetch/cli.py | 105 ++++++-- tariff_fetch/urdb/arcadia/__init__.py | 0 tariff_fetch/urdb/arcadia/build.py | 19 ++ tariff_fetch/urdb/arcadia/energyschedule.py | 189 +++++++++++++ tariff_fetch/urdb/arcadia/exception.py | 53 ++++ tariff_fetch/urdb/arcadia/library.py | 251 ++++++++++++++++++ tariff_fetch/urdb/arcadia/prompts.py | 163 ++++++++++++ tariff_fetch/urdb/arcadia/rateutils.py | 205 ++++++++++++++ tariff_fetch/urdb/arcadia/scenario.py | 18 ++ tariff_fetch/urdb/arcadia/shared.py | 68 +++++ tariff_fetch/urdb/arcadia/types.py | 9 + tariff_fetch/urdb/schema.py | 12 +- 16 files changed, 1070 insertions(+), 39 deletions(-) create mode 100644 tariff_fetch/urdb/arcadia/__init__.py create mode 100644 tariff_fetch/urdb/arcadia/build.py create mode 100644 tariff_fetch/urdb/arcadia/energyschedule.py create mode 100644 tariff_fetch/urdb/arcadia/exception.py create mode 100644 tariff_fetch/urdb/arcadia/library.py create mode 100644 tariff_fetch/urdb/arcadia/prompts.py create mode 100644 tariff_fetch/urdb/arcadia/rateutils.py create mode 100644 tariff_fetch/urdb/arcadia/scenario.py create mode 100644 tariff_fetch/urdb/arcadia/shared.py create mode 100644 tariff_fetch/urdb/arcadia/types.py diff --git a/pyproject.toml b/pyproject.toml index cc71c34..a731e65 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "tariff_fetch" -version = "0.1.1" +version = "0.2.0" description = "A CLI tool and a python library to simplify downloading electric and gas utility tariff data." authors = [{ name = "Switchbox", email = "hello@switch.box" }] readme = "README.md" @@ -70,7 +70,7 @@ fix = true [tool.ruff.lint] select = ["E", "W", "F", "B", "I", "UP", "SIM", "C4", "RUF"] -ignore = ["E501"] +ignore = ["E501", "SIM103"] [tool.ruff.lint.per-file-ignores] "tests/*" = ["S101"] diff --git a/tariff_fetch/_cli/genability.py b/tariff_fetch/_cli/genability.py index f5b57f3..680e4e6 100644 --- a/tariff_fetch/_cli/genability.py +++ b/tariff_fetch/_cli/genability.py @@ -1,5 +1,5 @@ import os -from datetime import date +from datetime import date, datetime from pathlib import Path from typing import cast @@ -124,7 +124,8 @@ def _fetch_tariffs(api: ArcadiaSignalAPI, tariffs: list[tuple[str, int]]): page = api.tariffs.iter_pages( fields="ext", master_tariff_id=id_, - effective_on=date.today(), + # effective_on=date.today(), + effective_on=date(2025, 6, 1), populate_properties=True, populate_rates=True, ) diff --git a/tariff_fetch/arcadia/api.py b/tariff_fetch/arcadia/api.py index a2b31c6..c86892b 100644 --- a/tariff_fetch/arcadia/api.py +++ b/tariff_fetch/arcadia/api.py @@ -85,6 +85,8 @@ class TariffsParams(TypedDict, total=False): populate_properties: bool populate_rates: bool populate_documents: bool + from_date_time: date + to_date_time: date class LookupsParams(PagingParams, total=False): diff --git a/tariff_fetch/arcadia/schema/tariffproperty.py b/tariff_fetch/arcadia/schema/tariffproperty.py index 22f0760..02f79b4 100644 --- a/tariff_fetch/arcadia/schema/tariffproperty.py +++ b/tariff_fetch/arcadia/schema/tariffproperty.py @@ -41,9 +41,9 @@ class TariffPropertyChoice(TypedDict): value: str "Machine readable option value" - displayValue: str + display_value: str "Human readable value shown to end users" - dataValue: str + data_value: str likelihood: NotRequired[float | None] @@ -62,7 +62,7 @@ class TariffPropertyMinimalFields(TypedDict): class TariffPropertyStandardFields(TypedDict): period: NotRequired[TariffPropertyPeriod] operator: str | None - propertyValue: NotRequired[str] + property_value: NotRequired[str] minValue: NotRequired[str | float | int] maxValue: NotRequired[str | float | int] choices: NotRequired[list[TariffPropertyChoice]] diff --git a/tariff_fetch/cli.py b/tariff_fetch/cli.py index 159785c..c970d0f 100644 --- a/tariff_fetch/cli.py +++ b/tariff_fetch/cli.py @@ -1,3 +1,5 @@ +import logging +from datetime import date from pathlib import Path from typing import Annotated, cast @@ -7,6 +9,7 @@ from requests import HTTPError from rich.prompt import Prompt +from tariff_fetch._cli.arcadia_urdb import process_genability as process_genability_urdb from tariff_fetch._cli.genability import process_genability from tariff_fetch._cli.openei import process_openei from tariff_fetch._cli.rateacuity import process_rateacuity @@ -21,6 +24,21 @@ ) +def prompt_year() -> int: + result = cast( + str, questionary.text("Enter year", default=str(date.today().year - 1), validate=_is_valid_year).ask() + ) + return int(result) + + +def _is_valid_year(value: str) -> bool: + try: + _ = date(int(value), 1, 1) + except (TypeError, ValueError): + return False + return True + + def prompt_state() -> StateCode: choice = Prompt.ask( "Enter two-letter state abbreviation", @@ -137,43 +155,74 @@ def main( output_folder: Annotated[ str, typer.Option("--output-folder", "-o", help="Folder to store outputs in") ] = "./outputs", + urdb: bool = False, ): + logging.basicConfig(level=logging.DEBUG) # print(pl.read_parquet(CoreEIA861_ASSN_UTILITY.https)) state_ = state or prompt_state().value providers_ = providers or prompt_providers() output_folder_ = Path(output_folder) utility = prompt_utility(state_) - if Provider.GENABILITY in providers_: - console.print("Processing [blue]Genability[/]") - try: - process_genability(utility=utility, output_folder=output_folder_) - except HTTPError as e: - if e.response.status_code == 401: + + if urdb: + year = prompt_year() + if Provider.GENABILITY in providers_: + console.print("Processing [blue]Genability[/]") + try: + process_genability_urdb(utility=utility, output_folder=output_folder_, year=year) + except HTTPError as e: + if e.response.status_code == 401: + console.print("Authorization failed") + console.print( + "Check if credentials set via [b]ARCADIA_APP_ID[/] and [b]ARCADIA_APP_KEY[/] environment variables are correct" + ) + else: + raise e from None + if Provider.OPENEI in providers_: + console.print("Processing [blue]OpenEI[/]") + try: + process_openei(utility, output_folder_) + except HTTPError as e: + if e.response.status_code == 403: + console.print("Authorization failed") + console.print("Check if [b]OPENEI_API_KEY[/] environment variable is correct") + else: + raise e from None + if Provider.RATEACUITY in providers_: + console.print("Cannot convert RateAcuity data to URDB") + + else: + if Provider.GENABILITY in providers_: + console.print("Processing [blue]Genability[/]") + try: + process_genability(utility=utility, output_folder=output_folder_) + except HTTPError as e: + if e.response.status_code == 401: + console.print("Authorization failed") + console.print( + "Check if credentials set via [b]ARCADIA_APP_ID[/] and [b]ARCADIA_APP_KEY[/] environment variables are correct" + ) + else: + raise e from None + if Provider.OPENEI in providers_: + console.print("Processing [blue]OpenEI[/]") + try: + process_openei(utility, output_folder_) + except HTTPError as e: + if e.response.status_code == 403: + console.print("Authorization failed") + console.print("Check if [b]OPENEI_API_KEY[/] environment variable is correct") + else: + raise e from None + if Provider.RATEACUITY in providers_: + console.print("Processing [blue]RateAcuity[/]") + try: + process_rateacuity(output_folder_, state_, utility) + except AuthorizationError: console.print("Authorization failed") console.print( - "Check if credentials set via [b]ARCADIA_APP_ID[/] and [b]ARCADIA_APP_KEY[/] environment variables are correct" + "Check if credentials provided via [b]RATEACUITY_USERNAME[/] and [b]RATEACUITY_PASSWORD[/] environment variables are correct" ) - else: - raise e from None - if Provider.OPENEI in providers_: - console.print("Processing [blue]OpenEI[/]") - try: - process_openei(utility, output_folder_) - except HTTPError as e: - if e.response.status_code == 403: - console.print("Authorization failed") - console.print("Check if [b]OPENEI_API_KEY[/] environment variable is correct") - else: - raise e from None - if Provider.RATEACUITY in providers_: - console.print("Processing [blue]RateAcuity[/]") - try: - process_rateacuity(output_folder_, state_, utility) - except AuthorizationError: - console.print("Authorization failed") - console.print( - "Check if credentials provided via [b]RATEACUITY_USERNAME[/] and [b]RATEACUITY_PASSWORD[/] environment variables are correct" - ) def main_cli(): diff --git a/tariff_fetch/urdb/arcadia/__init__.py b/tariff_fetch/urdb/arcadia/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tariff_fetch/urdb/arcadia/build.py b/tariff_fetch/urdb/arcadia/build.py new file mode 100644 index 0000000..c5013b8 --- /dev/null +++ b/tariff_fetch/urdb/arcadia/build.py @@ -0,0 +1,19 @@ +from dataclasses import dataclass + +from tariff_fetch.arcadia.api import ArcadiaSignalAPI +from tariff_fetch.urdb.arcadia.library import Library +from tariff_fetch.urdb.schema import URDBRate + +from .energyschedule import build_energy_schedule +from .scenario import Scenario + + +@dataclass(frozen=True) +class URDBChunk: + fixed_charge: float + + +def build_urdb(api: ArcadiaSignalAPI, scenario: Scenario) -> URDBRate: + library = Library(api) + energy_schedule = build_energy_schedule(scenario, library) + return {**energy_schedule} diff --git a/tariff_fetch/urdb/arcadia/energyschedule.py b/tariff_fetch/urdb/arcadia/energyschedule.py new file mode 100644 index 0000000..da79d0b --- /dev/null +++ b/tariff_fetch/urdb/arcadia/energyschedule.py @@ -0,0 +1,189 @@ +import itertools +from collections.abc import Collection +from datetime import datetime +from math import inf +from statistics import mean + +from tariff_fetch.arcadia.schema.common import RateChargeClass +from tariff_fetch.arcadia.schema.tariffrate import TariffRateExtended +from tariff_fetch.urdb.arcadia.library import Library +from tariff_fetch.urdb.schema import EnergyTier, URDBRate + +from . import rateutils as ru +from .exception import RateConversionError +from .scenario import Scenario +from .shared import is_weekday, is_weekend, iter_sampled_datetimes +from .types import ConsumptionBand, ConsumptionBandSet, DayPredicate + +PercentageModifiers = list[tuple[set[RateChargeClass], float]] +_ALLOWED_CHARGE_CLASSES: set[RateChargeClass] = {"DISTRIBUTION", "SUPPLY", "TRANSMISSION", "OTHER", "CONTRACTED"} + + +_RATE_PRECISION = 6 + + +def build_energy_schedule(scenario: Scenario, library: Library) -> URDBRate: + weekend_schedule_raw = [ + [get_month_hour_bands(scenario, library, month, hour, is_weekday) for hour in range(24)] + for month in range(1, 13) + ] + weekday_schedule_raw = [ + [get_month_hour_bands(scenario, library, month, hour, is_weekend) for hour in range(24)] + for month in range(1, 13) + ] + + # Order-preserving unique band sets + band_sets: list[tuple[ConsumptionBand, ...]] = [] + band_index: dict[tuple[ConsumptionBand, ...], int] = {} + + for schedule in (weekday_schedule_raw, weekend_schedule_raw): + for month in schedule: + for bands in month: + key = tuple(bands) + if key not in band_index: + band_index[key] = len(band_sets) + band_sets.append(key) + + energy_weekday_schedule = tuple(tuple(band_index[tuple(hour)] for hour in month) for month in weekday_schedule_raw) + energy_weekend_schedule = tuple(tuple(band_index[tuple(hour)] for hour in month) for month in weekend_schedule_raw) + + energy_rates_structure = [[_energy_band_to_tier(band) for band in bands] for bands in band_sets] + + return { + "energyratestructure": energy_rates_structure, + "energyweekdayschedule": energy_weekday_schedule, + "energyweekendschedule": energy_weekend_schedule, + } + + +def get_month_hour_bands( + scenario: Scenario, + library: Library, + month: int, + hour: int, + day_filter: DayPredicate, +) -> ConsumptionBandSet: + """Get raw consumption bands for specific month/hour""" + bands = [ + get_raw_bands_at_datetime(scenario, library, dt) + for dt in iter_sampled_datetimes(scenario.year, month, hour, day_filter) + ] + return average_aligned_bands(bands) + + +def get_raw_bands_at_datetime(scenario: Scenario, library: Library, dt: datetime) -> ConsumptionBandSet: + """Get raw tariff consumption-based bands at datetime dt""" + tariff = library.tariffs.get_tariff_at_date(scenario.master_tariff_id, dt) + rates = list(ru.tariff_iter_rates_for_dt(tariff, scenario, library, dt)) + percentage_modifiers = _get_percentage_rates_at_datetime(rates, scenario, library, dt) + piecewise_bands: list[ConsumptionBandSet] = [] + for rate in rates: + rate_bands = _get_rate_consumption_bands_at_datetime(rate, scenario, library, dt) + if rate_bands is None: + continue + # Apply percentage modifiers + charge_classes = set(rate.get("charge_class") or []) + charge_multiplier = 1.0 + sum(mv for mc, mv in percentage_modifiers if charge_classes & mc) + if scenario.apply_percentages: + rate_bands = [(limit, value * charge_multiplier) for limit, value in rate_bands] + piecewise_bands.append(rate_bands) + + return _sum_piecewise_bands(piecewise_bands) + + +def _get_rate_consumption_bands_at_datetime( + rate: TariffRateExtended, + scenario: Scenario, + library: Library, + dt: datetime, +) -> ConsumptionBandSet | None: + if rate.get("charge_type") != "CONSUMPTION_BASED": + return None + if set(rate.get("charge_class", [])) - _ALLOWED_CHARGE_CLASSES: + raise RateConversionError(rate, "Incorrect charge class for consumption-based rate") + if rate["charge_period"] != "MONTHLY": + raise RateConversionError(rate, "Incorrect charge period for consumption-based rate") + bands = ru.rate_filter_bands(rate, scenario, library) + + if any(band["rate_unit"] != "COST_PER_UNIT" for band in bands): + raise RateConversionError(rate, "Consumption bands must have rate unit = COST_PER_UNIT") + + return [ + (ru.band_consumption_upper_limit(band), ru.rate_band_get_amount_at_datetime(band, library, dt)) + for band in bands + ] + + +def _get_percentage_rates_at_datetime( + rates: Collection[TariffRateExtended], scenario: Scenario, library: Library, dt: datetime +) -> PercentageModifiers: + rates = [rate for rate in rates if ru.rate_get_band_units(rate) == {"PERCENTAGE"}] + + result: PercentageModifiers = [] + for rate in rates: + if not (bands := ru.rate_filter_bands(rate, scenario, library)): + raise RateConversionError(rate, "No bands for percentage rate") + + # if not (bands := ru.rate_filter_bands(rate, scenario)): + # raise RateConversionError(rate, "No bands for percentage rate") + if len(bands) > 1: + raise RateConversionError(rate, "Multiple bands are not supported for percentage rates") + if (charge_classes := rate.get("charge_class")) is None: + continue + band = bands[0] + if band["is_credit"]: + raise RateConversionError(rate, "Credit rates are not supported for pecentage rates") + + rate_amount = ru.rate_band_get_amount_at_datetime(band, library, dt) + result.append((set(charge_classes), rate_amount / 100.0)) + return result + + +def _sum_piecewise_bands(inputs: Collection[ConsumptionBandSet]) -> ConsumptionBandSet: + if not inputs: + return [] + + normalized = [sorted(bands, key=lambda b: b[0]) for bands in inputs] + limits = sorted({limit for bands in normalized for limit, _ in bands}) + + def value_at(bands: ConsumptionBandSet, limit: float) -> float: + for band_limit, value in bands: + if limit <= band_limit: + return value + return 0.0 + + summed = [(limit, sum(value_at(bands, limit) for bands in normalized)) for limit in limits] + + return [a for a, b in itertools.pairwise(summed) if a[1] != b[1]] + [summed[-1]] + + +def average_aligned_bands(inputs: list[ConsumptionBandSet]) -> ConsumptionBandSet: + if not inputs: + return [] + + normalized = [sorted(bands, key=lambda b: b[0]) for bands in inputs] + limits = sorted({limit for bands in normalized for limit, _ in bands}) + + def value_at(bands: ConsumptionBandSet, limit: float) -> float: + for band_limit, value in bands: + if limit <= band_limit: + return value + return 0.0 + + averaged = [ + (limit, round(mean(value_at(bands, limit) for bands in normalized), _RATE_PRECISION)) for limit in limits + ] + + # Collapse consecutive identical values + result = [averaged[0]] + for limit, value in averaged[1:]: + if value != result[-1][1]: + result.append((limit, value)) + + return result + + +def _energy_band_to_tier(band: ConsumptionBand) -> EnergyTier: + if band[0] == inf: + return {"rate": band[1], "unit": "kWh"} + return {"rate": band[1], "max": band[0], "unit": "kWh"} diff --git a/tariff_fetch/urdb/arcadia/exception.py b/tariff_fetch/urdb/arcadia/exception.py new file mode 100644 index 0000000..04eff99 --- /dev/null +++ b/tariff_fetch/urdb/arcadia/exception.py @@ -0,0 +1,53 @@ +from datetime import date +from typing import final +from typing_extensions import override + +from tariff_fetch.arcadia.schema.tariff import TariffExtended +from tariff_fetch.arcadia.schema.tariffrate import TariffRateExtended + + +class ConversionError(Exception): + pass + + +@final +class TariffConversionError(ConversionError): + def __init__(self, master_tariff_id: int, msg: str) -> None: + self.master_tariff_id = master_tariff_id + self.msg = msg + super().__init__(msg) + + +@final +class RateConversionError(ConversionError): + def __init__(self, rate: TariffRateExtended, msg: str) -> None: + self.rate = rate + self.msg = msg + super().__init__(msg) + + +class TariffNotFoundError(ConversionError): + pass + + +@final +class TariffNotFoundById(ConversionError): + def __init__(self, tariff_id: int) -> None: + self.tariff_id = tariff_id + super().__init__() + + @override + def __str__(self) -> str: + return f"Tariff with id={self.tariff_id} not found" + + +@final +class TariffNotFoundByDate(ConversionError): + def __init__(self, master_tariff_id: int, dt: date) -> None: + self.master_tariff_id = master_tariff_id + self.dt = dt + super().__init__() + + @override + def __str__(self) -> str: + return f"Tariff version not found for master_tariff_id={self.master_tariff_id} date={self.dt}" diff --git a/tariff_fetch/urdb/arcadia/library.py b/tariff_fetch/urdb/arcadia/library.py new file mode 100644 index 0000000..ac0a3e8 --- /dev/null +++ b/tariff_fetch/urdb/arcadia/library.py @@ -0,0 +1,251 @@ +from datetime import date, datetime +from typing import Literal, final, overload +from xdrlib import ConversionError + +from tariff_fetch.arcadia.api import ArcadiaSignalAPI +from tariff_fetch.arcadia.schema.lookup import Lookup +from tariff_fetch.arcadia.schema.tariff import TariffExtended +from tariff_fetch.arcadia.schema.tariffproperty import TariffPropertyPrunedDataType, TariffPropertyStandard +from tariff_fetch.arcadia.schema.tariffrate import TariffRateExtended +from tariff_fetch.urdb.arcadia.exception import TariffNotFoundByDate, TariffNotFoundById + +from .prompts import ( + prompt_boolean, + prompt_choice, + prompt_date, + prompt_decimal, + prompt_demand, + prompt_integer, + prompt_string, +) + +PropertyValue = str | list[str] | bool | date | float | int + + +@final +class TariffLibrary: + def __init__(self, api: ArcadiaSignalAPI) -> None: + self.api = api + self.tariffs: list[TariffExtended] = [] + + def get_tariff_at_date(self, master_tariff_id: int, dt: date) -> TariffExtended: + dt = dt.date() if isinstance(dt, datetime) else dt + if (found := self._find_tariff_at_date(master_tariff_id, dt)) is not None: + return found + tariff = self._fetch_tariff_at_date(master_tariff_id, dt) + self.tariffs.append(tariff) + return tariff + + def get_tariff(self, tariff_id: int) -> TariffExtended: + if (found := self._find_tariff(tariff_id)) is not None: + return found + tariff = self._fetch_tariff(tariff_id) + self.tariffs.append(tariff) + return tariff + + def get_rate(self, rate_id: int) -> TariffRateExtended: + return next( + rate for tariff in self.tariffs for rate in tariff.get("rates", []) if rate["tariff_rate_id"] == rate_id + ) + + def get_property(self, key: str) -> TariffPropertyStandard: + return next(prop for tariff in self.tariffs for prop in tariff.get("properties", []) if prop["key_name"] == key) + + def _find_tariff(self, tariff_id: int) -> TariffExtended | None: + return next((t for t in self.tariffs if t["tariff_id"] == tariff_id), None) + + def _find_tariff_at_date(self, master_tariff_id: int, dt: date) -> TariffExtended | None: + for tariff in self.tariffs: + if tariff["master_tariff_id"] != master_tariff_id: + continue + if _is_tariff_effective_on(tariff, dt): + return tariff + return None + + def _fetch_tariff(self, tariff_id: int) -> TariffExtended: + tariffs = list( + self.api.tariffs.iter_pages( + fields="ext", + search=str(tariff_id), + search_on=["tariffId"], + populate_properties=True, + populate_rates=True, + ) + ) + if len(tariffs) > 1: + raise RuntimeError(f"More than one tariff found for id={tariff_id}") + if not tariffs: + raise TariffNotFoundById(tariff_id) + return tariffs[0] + + def _fetch_tariff_at_date(self, master_tariff_id: int, dt: date) -> TariffExtended: + tariffs = self.api.tariffs.iter_pages( + fields="ext", + master_tariff_id=master_tariff_id, + effective_on=(dt.date() if isinstance(dt, datetime) else dt), + populate_properties=True, + populate_rates=True, + ) + try: + return next(t for t in tariffs if _is_tariff_effective_on(t, dt)) + except StopIteration as e: + raise TariffNotFoundByDate(master_tariff_id, dt) from e + + +@final +class VariablePropertyLibrary: + def __init__(self, api: ArcadiaSignalAPI): + self.api = api + self.property_timeseries: dict[tuple[str, int], list[Lookup]] = {} + + def lookup(self, key: str, dt: datetime) -> float: + if (lookups := self.property_timeseries.get((key, dt.year))) is None: + lookups = self._lookup_property_timeseries(key, dt.year) + self.property_timeseries[(key, dt.year)] = lookups + + for row in lookups: + if row["from_date_time"] <= dt <= (row["to_date_time"] or datetime.max): + if (value := row["actual_value"]) is not None: + return value + if (value := row["best_value"]) is not None: + return value + if (value := row["forecast_value"]) is not None: + return value + return 0.0 + return 0.0 + + def _lookup_property_timeseries(self, key: str, year: int) -> list[Lookup]: + return list( + self.api.properties.lookups.iter_pages( + key, + from_date_time=date(year, 1, 1), + to_date_time=date(year + 1, 1, 1), + ) + ) + + +@final +class Library: + def __init__(self, api: ArcadiaSignalAPI, properties: dict[str, PropertyValue] | None = None): + self.api = api + self.tariffs = TariffLibrary(api) + self.variables = VariablePropertyLibrary(api) + self._properies: dict[str, PropertyValue] = properties or {} + + def has_property(self, key: str) -> bool: + return key in self._properies + + def get_choice_property_as_ints(self, key: str) -> list[int]: + strs = self.get_property(key, "CHOICE") + try: + return list(map(int, strs)) + except ValueError as e: + raise ConversionError(f"Could not convert value of choice property {key} to strings") from e + + @overload + def get_property(self, key: str, data_type: Literal["STRING"]) -> str: ... + + @overload + def get_property(self, key: str, data_type: Literal["CHOICE"]) -> list[str]: ... + + @overload + def get_property(self, key: str, data_type: Literal["BOOLEAN"]) -> bool: ... + @overload + def get_property(self, key: str, data_type: Literal["DATE"]) -> date: ... + @overload + def get_property(self, key: str, data_type: Literal["DECIMAL"]) -> float: ... + @overload + def get_property(self, key: str, data_type: Literal["INTEGER"]) -> int: ... + @overload + def get_property(self, key: str, data_type: Literal["DEMAND"]) -> float: ... + + def get_property(self, key: str, data_type: TariffPropertyPrunedDataType) -> PropertyValue: + if (found := self._get_property(key, data_type)) is not None: + return found + tariff_property = self.tariffs.get_property(key) + result = _prompt_property(tariff_property) + if result is None: + raise ConversionError("Property not set") + self._properies[key] = result + return result + + @overload + def _get_property(self, key: str, data_type: Literal["STRING"]) -> str | None: ... + + @overload + def _get_property(self, key: str, data_type: Literal["CHOICE"]) -> list[str] | None: ... + + @overload + def _get_property(self, key: str, data_type: Literal["BOOLEAN"]) -> bool | None: ... + @overload + def _get_property(self, key: str, data_type: Literal["DATE"]) -> date | None: ... + @overload + def _get_property(self, key: str, data_type: Literal["DECIMAL"]) -> float | None: ... + @overload + def _get_property(self, key: str, data_type: Literal["INTEGER"]) -> int | None: ... + @overload + def _get_property(self, key: str, data_type: Literal["DEMAND"]) -> float | None: ... + + def _get_property(self, key: str, data_type: TariffPropertyPrunedDataType) -> PropertyValue | None: + try: + value = self._properies[key] + except KeyError: + return None + value_type = type(value) + match data_type: + case "STRING": + if not isinstance(value, str): + raise ConversionError(f"Value for property {key} is expected to be `str`, not {value_type}") + return value + case "CHOICE": + if not isinstance(value, list): + raise ConversionError(f"Value for property {key} is expected to be a list, not {value_type}") + return value + case "BOOLEAN": + if not isinstance(value, bool): + raise ConversionError(f"Value for property {key} is expected to be `bool`, not {value_type}") + return value + case "DATE": + if not isinstance(value, date): + raise ConversionError(f"Value for property {key} is expected to be `bool`, not {value_type}") + return value + case "DECIMAL": + if not isinstance(value, float): + raise ConversionError(f"Value for property {key} is expected to be `float`, not {value_type}") + return value + case "INTEGER": + if not isinstance(value, bool): + raise ConversionError(f"Value for property {key} is expected to be `int`, not {value_type}") + return value + + case "DEMAND": + if not isinstance(value, bool): + raise ConversionError(f"Value for property {key} is expected to be `float`, not {value_type}") + return value + + +def _is_tariff_effective_on(tariff: TariffExtended, dt: date) -> bool: + effective_date = tariff["effective_date"] + end_date = tariff["end_date"] or date.max + return effective_date <= dt < end_date + + +def _prompt_property(tariff_property: TariffPropertyStandard) -> PropertyValue | None: + data_type = tariff_property["data_type"] + match data_type: + case "BOOLEAN": + return prompt_boolean(tariff_property) + case "CHOICE": + return prompt_choice(tariff_property) + case "STRING": + return prompt_string(tariff_property) + case "DATE": + return prompt_date(tariff_property) + case "DECIMAL": + return prompt_decimal(tariff_property) + case "INTEGER": + return prompt_integer(tariff_property) + case "DEMAND": + return prompt_demand(tariff_property) + case "FORMULA": + raise ValueError("Formula properties are not supported") diff --git a/tariff_fetch/urdb/arcadia/prompts.py b/tariff_fetch/urdb/arcadia/prompts.py new file mode 100644 index 0000000..b08bad3 --- /dev/null +++ b/tariff_fetch/urdb/arcadia/prompts.py @@ -0,0 +1,163 @@ +from datetime import date +from typing import cast, get_args + +import questionary + +from tariff_fetch.arcadia.schema.common import RateChargeClass +from tariff_fetch.arcadia.schema.tariffproperty import TariffPropertyStandard + + +# def prompt_scenario(master_tariff_id: int, year: int, apply_percentages: bool) -> Scenario | None: +# result_charge_classes = prompt_charge_classes() +# if result_charge_classes is None: +# return None +# +# properties = tariff.get("properties", []) +# result_property_values: dict[str, PropertyValues] = {} +# for tariff_property in properties: +# key = tariff_property["key_name"] +# if key == "chargeClass": +# continue +# if key == "consumption": +# continue +# value = prompt_property(tariff_property) +# if value is None: +# return None +# result_property_values[key] = value +# return Scenario( +# tariff, +# year=year, +# apply_percentages=apply_percentages, +# charge_classes=result_charge_classes, +# properies=result_property_values, +# ) + + +def prompt_charge_classes() -> set[RateChargeClass] | None: + choices = cast(tuple[RateChargeClass, ...], get_args(RateChargeClass)) + result_raw = cast( + list[RateChargeClass] | None, + questionary.checkbox( + "Select charge classes", + choices=[ + questionary.Choice( + title=choice, + value=choice, + checked=True, + ) + for choice in choices + ], + ).ask(), + ) + if result_raw is None: + return None + return set(result_raw) + + +def prompt_string(tariff_property: TariffPropertyStandard) -> str | None: + default_value = tariff_property.get("property_value") if tariff_property["is_default"] else None + return cast( + str | None, + questionary.text( + _get_property_msg(tariff_property), + default=default_value or "", + ).ask(), + ) + + +def prompt_choice(tariff_property: TariffPropertyStandard) -> list[str] | None: + if tariff_property["is_default"]: + default_value_raw = tariff_property.get("property_value") + default_value = {item.strip() for item in (default_value_raw or "").split(",")} + else: + default_value: set[str] = set() + if not (choices := tariff_property.get("choices")): + raise ValueError("Expected a list of choices for CHOICE property") + return cast( + list[str] | None, + questionary.checkbox( + _get_property_msg(tariff_property), + choices=[ + questionary.Choice( + title=item["display_value"], value=item["value"], checked=item["value"] in default_value + ) + for item in choices + ], + ).ask(), + ) + + +def prompt_boolean(tariff_property: TariffPropertyStandard) -> bool | None: + default_value = tariff_property.get("property_value") if tariff_property["is_default"] else None + default_value = True if default_value == "true" else (False if default_value == "false" else None) + result = cast( + bool | None, + questionary.confirm( + _get_property_msg(tariff_property), default=default_value if default_value is not None else False + ).ask(), + ) + return result + + +def prompt_date(tariff_property: TariffPropertyStandard) -> date | None: + raise NotImplementedError() + + +def prompt_decimal(tariff_property: TariffPropertyStandard) -> float | None: + default_value = tariff_property.get("property_value") if tariff_property["is_default"] else None + default_value = float(default_value) if default_value else None + + result_str = cast( + str | None, + questionary.text( + _get_property_msg(tariff_property), + default=str(default_value) if default_value else "", + validate=_is_float, + ).ask(), + ) + if result_str is None: + return None + return float(result_str) + + +def prompt_integer(tariff_property: TariffPropertyStandard) -> float | None: + default_value = tariff_property.get("property_value") if tariff_property["is_default"] else None + default_value = int(default_value) if default_value else None + + result_str = cast( + str | None, + questionary.text( + _get_property_msg(tariff_property), + default=str(default_value) if default_value else "", + validate=_is_int, + ).ask(), + ) + if result_str is None: + return None + return float(result_str) + + +def prompt_demand(tariff_property: TariffPropertyStandard) -> float | None: + return prompt_decimal(tariff_property) + + +def _is_float(value: str) -> bool: + try: + _ = float(value) + except ValueError: + return False + return True + + +def _is_int(value: str) -> bool: + try: + _ = int(value) + except ValueError: + return False + return True + + +def _get_property_msg(tariff_property: TariffPropertyStandard) -> str: + title = tariff_property["display_name"] + description = tariff_property["description"] + return f"{title} ({description})" diff --git a/tariff_fetch/urdb/arcadia/rateutils.py b/tariff_fetch/urdb/arcadia/rateutils.py new file mode 100644 index 0000000..24025e5 --- /dev/null +++ b/tariff_fetch/urdb/arcadia/rateutils.py @@ -0,0 +1,205 @@ +from collections.abc import Collection, Iterator +from datetime import date, datetime +from math import inf + +from tariff_fetch.arcadia.api import ArcadiaSignalAPI +from tariff_fetch.arcadia.schema.common import RateChargeClass, RateUnit +from tariff_fetch.arcadia.schema.season import SeasonExtended +from tariff_fetch.arcadia.schema.tariff import TariffExtended +from tariff_fetch.arcadia.schema.tariffrate import TariffRateBand, TariffRateExtended +from tariff_fetch.arcadia.schema.timeofuse import Period, TimeOfUseExtended +from tariff_fetch.urdb.arcadia.library import Library, TariffLibrary, VariablePropertyLibrary +from tariff_fetch.urdb.arcadia.shared import lookup_variable_rate + +from .exception import RateConversionError, TariffConversionError +from .scenario import Scenario + +# ================================ +# Tariff +# ================================ + + +def tariff_iter_rates_for_dt( + tariff: TariffExtended, + scenario: Scenario, + library: Library, + dt: datetime, +) -> Iterator[TariffRateExtended]: + rates = tariff.get("rates", []) + for rate in rates: + if not rate_is_applied_to_scenario(rate, scenario, library): + continue + if not rate_is_applied_to_datetime(rate, dt): + continue + if rate["rate_bands"]: + yield rate + elif rider_id := rate.get("rider_id"): + rider_tariff = library.tariffs.get_tariff(rider_id) + yield from tariff_iter_rates_for_dt(rider_tariff, scenario, library, dt) + + +# ================================ +# Rate +# ================================ + + +def rate_is_applied_to_scenario( + rate: TariffRateExtended, + scenario: Scenario, + library: Library, +) -> bool: + if not rate_is_applied_to_charge_classes(rate, scenario.charge_classes): + return False + if (territory := rate.get("territory")) is not None: + territory_id = territory["territory_id"] + territory_ids = library.get_choice_property_as_ints("territoryId") + if territory_id not in territory_ids: + return False + + return True + + +def rate_is_applied_to_charge_classes(rate: TariffRateExtended, charge_classes: Collection[RateChargeClass]) -> bool: + if (rate_charge_classes := rate.get("charge_class")) is None: + return True + return not set(rate_charge_classes).isdisjoint(charge_classes) + + +def rate_is_applied_to_datetime(rate: TariffRateExtended, dt: datetime) -> bool: + if (season := rate.get("season")) and not season_is_datetime_within(season, dt): + return False + if (tou := rate.get("time_of_use")) and not time_of_use_is_datetime_within(tou, dt): + return False + return True + + +def rate_filter_bands(rate: TariffRateExtended, scenario: Scenario, library: Library) -> list[TariffRateBand]: + result: list[TariffRateBand] = [] + for band in rate.get("rate_bands"): + if band.get("has_demand_limit"): + raise RateConversionError(rate, "Bands with demand limits are not supported") + if band.get("demand_upper_limit"): + raise RateConversionError(rate, "Bands with demand limits are not supported") + if band.get("has_property_limit"): + raise RateConversionError(rate, "Bands with property limits are not supported") + if band.get("property_upper_limit"): + raise RateConversionError(rate, "Bands with property limits are not supported") + if band.get("prev_upper_limit"): + raise RateConversionError(rate, "Bands with property prev_upper_limit are not supported") + if band.get("calculation_factor"): + raise RateConversionError(rate, "Bands with property calculation_factor are not supported") + if band.get("applicability_formula"): + raise RateConversionError(rate, "Bands with property applicability_formula are not supported") + if (applicability_value := band.get("applicability_value")) is not None: + if (applicability_key := rate.get("applicability_key")) is None: + raise RateConversionError(rate, "Band has applicability value but rate doesn't have applicability key") + tariff_property = library.tariffs.get_property(applicability_key) + + if (tariff_property.get("period")) is not None: + raise RateConversionError(rate, "Period properties are not supported for rate bands") + + if (category := tariff_property["property_types"]) not in {"APPLICABILITY", "RATE_CRITERIA"}: + raise RateConversionError(rate, f"{category} is not a supported property category for tariff bands") + + operator = tariff_property["operator"] + if operator != "=": + raise RateConversionError(rate, "Only `=` operators are supported") + match tariff_property["data_type"]: + case "CHOICE": + if applicability_value not in library.get_property(applicability_key, "CHOICE"): + continue + case "STRING": + if applicability_value != library.get_property(applicability_value, "STRING"): + continue + case "BOOLEAN": + match applicability_value: + case "true": + applicability_value_bool = True + case "false": + applicability_value_bool = False + case _: + raise RateConversionError( + rate, "Boolean applicability value must be either `true` or `false`" + ) + if applicability_value_bool != library.get_property(applicability_key, "BOOLEAN"): + continue + case _: + raise RateConversionError(rate, "Unsupported data type for band applicabiltiy properties") + result.append(band) + return result + + +def rate_get_band_units(rate: TariffRateExtended) -> set[RateUnit]: + return {b["rate_unit"] for b in rate.get("rate_bands", [])} + + +# ================================ +# Rate Band +# ================================ +def band_consumption_upper_limit(band: TariffRateBand) -> float: + return band.get("consumption_upper_limit") or inf + + +def rate_band_get_amount_at_datetime(band: TariffRateBand, library: Library, dt: datetime) -> float: + rate_id = band["tariff_rate_id"] + rate = library.tariffs.get_rate(rate_id) + mp = -1 if band["is_credit"] else 1 + if (variable_rate_key := rate.get("variable_rate_key")) is None: + return band["rate_amount"] * mp + return library.variables.lookup(variable_rate_key, dt) * mp + + +# ================================ +# Time of Use +# ================================ + + +def time_of_use_is_datetime_within( + tou: TimeOfUseExtended, + dt: datetime, +) -> bool: + season = tou.get("season") + if season and not season_is_datetime_within(season, dt): + return False + + return any(period_is_datetime_within(period, dt) for period in tou["tou_periods"]) + + +# ================================ +# Time of Use - Period +# ================================ + + +def period_is_datetime_within( + period: Period, + dt: datetime, +) -> bool: + return ( + period["from_day_of_week"] <= dt.weekday() <= period["to_day_of_week"] + and period["from_hour"] <= dt.hour <= period["to_hour"] + and period["from_minute"] <= dt.minute <= period["to_minute"] + ) + + +# ================================ +# Season +# ================================ + + +def season_is_datetime_within( + season: SeasonExtended, + dt: datetime | date, +) -> bool: + dt = dt.date() if isinstance(dt, datetime) else dt + start_month, end_month = season["season_from_month"], season["season_to_month"] + start_day, end_day = season["season_from_day"], season["season_to_day"] + start_date = date(dt.year, start_month, start_day) + end_date = date(dt.year, end_month, end_day) + if start_date < end_date: + return start_date <= dt < end_date + return start_date <= dt or dt < end_date + + +# ================================ +# Rate +# ================================ diff --git a/tariff_fetch/urdb/arcadia/scenario.py b/tariff_fetch/urdb/arcadia/scenario.py new file mode 100644 index 0000000..8d89e69 --- /dev/null +++ b/tariff_fetch/urdb/arcadia/scenario.py @@ -0,0 +1,18 @@ +from dataclasses import dataclass, field +from datetime import date +from typing import Literal, cast, get_args, overload + +from tariff_fetch.arcadia.schema.common import RateChargeClass +from tariff_fetch.arcadia.schema.tariff import TariffExtended +from tariff_fetch.arcadia.schema.tariffproperty import TariffPropertyPrunedDataType +from tariff_fetch.urdb.arcadia.exception import TariffConversionError + +_CHARGE_CLASSES = cast(tuple[RateChargeClass, ...], get_args(RateChargeClass)) + + +@dataclass(frozen=True) +class Scenario: + master_tariff_id: int + year: int + apply_percentages: bool + charge_classes: set[RateChargeClass] = field(default_factory=set) diff --git a/tariff_fetch/urdb/arcadia/shared.py b/tariff_fetch/urdb/arcadia/shared.py new file mode 100644 index 0000000..1fb1e63 --- /dev/null +++ b/tariff_fetch/urdb/arcadia/shared.py @@ -0,0 +1,68 @@ +import calendar +from datetime import date, datetime +from functools import lru_cache + +from tariff_fetch.arcadia.api import ArcadiaSignalAPI + +from .types import DayPredicate + + +def iter_sampled_datetimes( + year: int, + month: int, + hour: int, + day_filter: DayPredicate, +): + _, days = calendar.monthrange(year, month) + + for day in range(1, days + 1): + calendar_day = datetime(year, month, day) + if day_filter(calendar_day): + yield sample_datetime(year, month, day, hour) + + +def sample_datetime(year: int, month: int, day: int, hour: int) -> datetime: + """ + Representative instant for tariff evaluation. + + We intentionally sample at hh:30 to avoid boundary effects and + because Arcadia TOU periods are hour-granular. + """ + return datetime(year, month, day, hour, 30) + + +def is_weekday(dt: datetime) -> bool: + return dt.weekday() < 5 + + +def is_weekend(dt: datetime) -> bool: + return dt.weekday() >= 5 + + +def lookup_variable_rate( + api: ArcadiaSignalAPI, + key: str, + dt: datetime, +) -> float: + lookups = lookup_property_timeseries(api, key, dt.year) + for row in lookups: + if row["from_date_time"] <= dt <= (row["to_date_time"] or datetime.max): + if (value := row["actual_value"]) is not None: + return value + if (value := row["best_value"]) is not None: + return value + if (value := row["forecast_value"]) is not None: + return value + return 0.0 + return 0.0 + + +@lru_cache(maxsize=256) +def lookup_property_timeseries(api: ArcadiaSignalAPI, key: str, year: int): + return list( + api.properties.lookups.iter_pages( + key, + from_date_time=date(year, 1, 1), + to_date_time=date(year + 1, 1, 1), + ) + ) diff --git a/tariff_fetch/urdb/arcadia/types.py b/tariff_fetch/urdb/arcadia/types.py new file mode 100644 index 0000000..17a8e39 --- /dev/null +++ b/tariff_fetch/urdb/arcadia/types.py @@ -0,0 +1,9 @@ +from collections.abc import Callable +from datetime import datetime + +from tariff_fetch.arcadia.schema.common import RateChargeClass + +ConsumptionBand = tuple[float, float] # (upper_limit, value) +ConsumptionBandSet = list[ConsumptionBand] +DayPredicate = Callable[[datetime], bool] +PercentageModifiers = list[tuple[set[RateChargeClass], float]] diff --git a/tariff_fetch/urdb/schema.py b/tariff_fetch/urdb/schema.py index 516de10..1063943 100644 --- a/tariff_fetch/urdb/schema.py +++ b/tariff_fetch/urdb/schema.py @@ -29,7 +29,7 @@ # --- fixed-shape schedules (12 months x 24 hours) --- -Int24 = tuple[ +Int24_ = tuple[ int, int, int, @@ -55,9 +55,13 @@ int, int, ] -MonthSchedule = tuple[Int24, Int24, Int24, Int24, Int24, Int24, Int24, Int24, Int24, Int24, Int24, Int24] -Float12 = tuple[float, float, float, float, float, float, float, float, float, float, float, float] -FlatDemandMonths = tuple[int, int, int, int, int, int, int, int, int, int, int, int] +Int24 = tuple[int, ...] +# MonthSchedule = tuple[Int24, Int24, Int24, Int24, Int24, Int24, Int24, Int24, Int24, Int24, Int24, Int24] +MonthSchedule = tuple[Int24, ...] +# Float12 = tuple[float, float, float, float, float, float, float, float, float, float, float, float] +Float12 = tuple[float, ...] +# FlatDemandMonths = tuple[int, int, int, int, int, int, int, int, int, int, int, int] +FlatDemandMonths = tuple[int, ...] # --- tiers (NO None allowed) --- From 6d065476dbec701c432593ed6ccc1e76521f9676 Mon Sep 17 00:00:00 2001 From: Natalia Borovskikh Date: Tue, 3 Mar 2026 13:06:35 +0000 Subject: [PATCH 02/24] bump version --- uv.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uv.lock b/uv.lock index 23a4d36..faf8e64 100644 --- a/uv.lock +++ b/uv.lock @@ -1480,7 +1480,7 @@ wheels = [ [[package]] name = "tariff-fetch" -version = "0.1" +version = "0.2.0" source = { editable = "." } dependencies = [ { name = "fastexcel" }, From 585120845ed26fb81e75fcc0dd6590b9ab830500 Mon Sep 17 00:00:00 2001 From: Natalia Borovskikh Date: Wed, 4 Mar 2026 12:23:39 +0000 Subject: [PATCH 03/24] bump version --- pyproject.toml | 2 +- tariff_fetch/urdb/arcadia/build.py | 28 ++++++++++++++++++++++++++-- uv.lock | 2 +- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a731e65..e585290 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "tariff_fetch" -version = "0.2.0" +version = "0.3.0" description = "A CLI tool and a python library to simplify downloading electric and gas utility tariff data." authors = [{ name = "Switchbox", email = "hello@switch.box" }] readme = "README.md" diff --git a/tariff_fetch/urdb/arcadia/build.py b/tariff_fetch/urdb/arcadia/build.py index c5013b8..0a762cb 100644 --- a/tariff_fetch/urdb/arcadia/build.py +++ b/tariff_fetch/urdb/arcadia/build.py @@ -1,10 +1,14 @@ from dataclasses import dataclass +from typing import cast + +import questionary from tariff_fetch.arcadia.api import ArcadiaSignalAPI from tariff_fetch.urdb.arcadia.library import Library from tariff_fetch.urdb.schema import URDBRate from .energyschedule import build_energy_schedule +from .fixedcharge import build_fixed_charge from .scenario import Scenario @@ -15,5 +19,25 @@ class URDBChunk: def build_urdb(api: ArcadiaSignalAPI, scenario: Scenario) -> URDBRate: library = Library(api) - energy_schedule = build_energy_schedule(scenario, library) - return {**energy_schedule} + try: + energy_schedule = build_energy_schedule(scenario, library) + except Exception as e: + energy_schedule = _confirm_proceed(e, "energy rate strucutre") + + try: + fixed_charge = build_fixed_charge(scenario, library) + except Exception as e: + fixed_charge = _confirm_proceed(e, "fixed charges") + return {**energy_schedule, **fixed_charge} + + +def _confirm_proceed(e: Exception, processing: str) -> URDBRate: + response = cast( + bool | None, + questionary.confirm(f"Error while converting: {processing}: {e}. Continue or print traceback and exit?").ask(), + ) + if response is None: + exit() + if response: + return {} + raise e from None diff --git a/uv.lock b/uv.lock index faf8e64..31a998b 100644 --- a/uv.lock +++ b/uv.lock @@ -1480,7 +1480,7 @@ wheels = [ [[package]] name = "tariff-fetch" -version = "0.2.0" +version = "0.3.0" source = { editable = "." } dependencies = [ { name = "fastexcel" }, From f2bd5b85c96fa789dd1f1b1fda1e0c2e3203be2d Mon Sep 17 00:00:00 2001 From: Natalia Borovskikh Date: Wed, 4 Mar 2026 12:24:15 +0000 Subject: [PATCH 04/24] some cleanup --- tariff_fetch/urdb/arcadia/rateutils.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/tariff_fetch/urdb/arcadia/rateutils.py b/tariff_fetch/urdb/arcadia/rateutils.py index 24025e5..d13a07e 100644 --- a/tariff_fetch/urdb/arcadia/rateutils.py +++ b/tariff_fetch/urdb/arcadia/rateutils.py @@ -2,16 +2,14 @@ from datetime import date, datetime from math import inf -from tariff_fetch.arcadia.api import ArcadiaSignalAPI from tariff_fetch.arcadia.schema.common import RateChargeClass, RateUnit from tariff_fetch.arcadia.schema.season import SeasonExtended from tariff_fetch.arcadia.schema.tariff import TariffExtended from tariff_fetch.arcadia.schema.tariffrate import TariffRateBand, TariffRateExtended from tariff_fetch.arcadia.schema.timeofuse import Period, TimeOfUseExtended -from tariff_fetch.urdb.arcadia.library import Library, TariffLibrary, VariablePropertyLibrary -from tariff_fetch.urdb.arcadia.shared import lookup_variable_rate +from tariff_fetch.urdb.arcadia.library import Library -from .exception import RateConversionError, TariffConversionError +from .exception import RateConversionError from .scenario import Scenario # ================================ @@ -198,8 +196,3 @@ def season_is_datetime_within( if start_date < end_date: return start_date <= dt < end_date return start_date <= dt or dt < end_date - - -# ================================ -# Rate -# ================================ From fdea7b56a33d31a66065d4149032eeae6c3bb6f4 Mon Sep 17 00:00:00 2001 From: Natalia Borovskikh Date: Wed, 4 Mar 2026 13:10:22 +0000 Subject: [PATCH 05/24] Add metadata and fixed charges --- pyproject.toml | 2 +- tariff_fetch/_cli/arcadia_urdb.py | 208 +++++++++++++++++++++++ tariff_fetch/urdb/arcadia/build.py | 9 +- tariff_fetch/urdb/arcadia/fixedcharge.py | 74 ++++++++ tariff_fetch/urdb/arcadia/metadata.py | 12 ++ uv.lock | 2 +- 6 files changed, 304 insertions(+), 3 deletions(-) create mode 100644 tariff_fetch/_cli/arcadia_urdb.py create mode 100644 tariff_fetch/urdb/arcadia/fixedcharge.py create mode 100644 tariff_fetch/urdb/arcadia/metadata.py diff --git a/pyproject.toml b/pyproject.toml index e585290..6e83252 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "tariff_fetch" -version = "0.3.0" +version = "0.4.0" description = "A CLI tool and a python library to simplify downloading electric and gas utility tariff data." authors = [{ name = "Switchbox", email = "hello@switch.box" }] readme = "README.md" diff --git a/tariff_fetch/_cli/arcadia_urdb.py b/tariff_fetch/_cli/arcadia_urdb.py new file mode 100644 index 0000000..3cdf7a4 --- /dev/null +++ b/tariff_fetch/_cli/arcadia_urdb.py @@ -0,0 +1,208 @@ +import json +import os +from datetime import date +from pathlib import Path +from typing import cast, get_args + +import questionary +from dotenv import load_dotenv +from pydantic import TypeAdapter + +# from tariff_fetch.genability.lse import get_lses_page +# from tariff_fetch.genability.tariffs import CustomerClass, TariffType, tariffs_paginate +from tariff_fetch.arcadia.api import ArcadiaSignalAPI +from tariff_fetch.arcadia.schema.common import CustomerClass, RateChargeClass, TariffType +from tariff_fetch.arcadia.schema.tariff import TariffExtended +from tariff_fetch.urdb.arcadia.build import build_urdb +from tariff_fetch.urdb.arcadia.prompts import prompt_charge_classes +from tariff_fetch.urdb.arcadia.scenario import Scenario +from tariff_fetch.urdb.schema import URDBRate + +from . import console, prompt_filename +from .types import Utility + + +def process_genability(utility: Utility, output_folder: Path, year: int): + _ = load_dotenv() + if not os.getenv("ARCADIA_APP_ID"): + console.print("[b]ARCADIA_APP_ID[/] environment variable is not set.") + if not os.getenv("ARCADIA_APP_KEY"): + console.print("[b]ARCADIA_APP_KEY[/] environment variable is not set.") + if not (os.getenv("ARCADIA_APP_ID") and os.getenv("ARCADIA_APP_KEY")): + console.print("Cannot use Arcadia API due to missing credentials") + _ = console.input("Press enter to proceed...") + return + api = ArcadiaSignalAPI() + + lse_id = _find_utility_lse_id(api, utility) + if lse_id is None: + return + + if not (customer_classes := _select_customer_classes()): + return + + if not (tariff_types := _select_tariff_types()): + return + + if not (tariffs := _select_tariffs(api, lse_id, customer_classes, tariff_types, year)): + console.print("[red]No tariffs found[/]") + _ = console.input("Press enter to proceed...") + return + + api_results = _fetch_tariffs(api, tariffs, year) + results: list[URDBRate] = [] + + # tariff_ids = {id_ for _, id_ in tariffs} + + apply_percentages = cast(bool | None, questionary.confirm("Apply percentage rates?").ask()) + if apply_percentages is None: + return + + for tariff in api_results: + tariff_name = tariff["tariff_name"] + console.print(f"Creating scenario for tariff: [blue]{tariff_name}[/]") + charge_classes = prompt_charge_classes() + if charge_classes is None: + continue + scenario = Scenario( + tariff["master_tariff_id"], + year=year, + apply_percentages=apply_percentages, + charge_classes=charge_classes, + ) + urdb_tariff = build_urdb(api, scenario) + urdb_tariff["name"] = _prompt_tariff_name(urdb_tariff.get("name", "")) + + results.append(urdb_tariff) + # results.append(build_urdb(tariff, api, scenario)) + + suggested_filename = f"arcadia_{utility.name}.urdb.{year}." + + if not (filename := prompt_filename(output_folder, suggested_filename, "json")): + return + + filename.parent.mkdir(exist_ok=True) + _ = filename.write_text(json.dumps({"items": results}, indent=2)) + console.print(f"Wrote [blue]{len(results)}[/] records to {filename}") + + +def _find_utility_lse_id(api: ArcadiaSignalAPI, utility: Utility) -> int | None: + with console.status("Fetching lses..."): + lses = api.lses.get_page( + fields="min", + search_on=["code"], + search=str(utility.eia_id), + starts_with=True, + ends_with=True, + )["results"] + if len(lses) == 0: + # No utilities found with this eia id + console.print( + f'Utility "{utility.name}" with EIA Id {utility.eia_id} not found in arcadia database', style="bold red" + ) + return None + if len(lses) == 1: + # Found one utility + utility_lse_id = lses[0]["lse_id"] + return utility_lse_id + else: + # Nothing found; this should *theoretically* never happen but let's keep it just in case + choices = [questionary.Choice(title=_["name"], value=_["lse_id"]) for _ in lses] + choices.append(questionary.Separator()) + choices.append(questionary.Choice(title="None of these", value=None)) + utility_lse_id = cast( + int | None, + questionary.select( + message=f"Found multiple utilities with lse id = {utility.eia_id}. Select one.", choices=choices + ).ask(), + ) + if utility_lse_id is None: + console.print("No utility chosen") + return None + return utility_lse_id + + +def _select_tariffs( + api: ArcadiaSignalAPI, lse_id: int, customer_classes: list[CustomerClass], tariff_types: list[TariffType], year: int +) -> list[tuple[str, int]]: + with console.status("Fetching tariffs..."): + tariffs = list( + api.tariffs.iter_pages( + lse_id=lse_id, + effective_on=date(year, 6, 1), + customer_classes=customer_classes, + tariff_types=tariff_types, + ) + ) + if not tariffs: + return [] + return cast( + list[tuple[str, int]], + questionary.checkbox( + message="Select tariffs", + choices=[ + questionary.Choice( + title=f"{_['tariff_name']} ({_['tariff_id']})", + value=(_["tariff_name"], _["master_tariff_id"]), # pyright: ignore[reportAny] + checked=True, + ) + for _ in tariffs + ], + use_search_filter=True, + use_jk_keys=False, + ).ask(), + ) + + +def _select_customer_classes() -> list[CustomerClass]: + return cast( + list[CustomerClass], + questionary.checkbox( + message="Select customer classes", + choices=[ + questionary.Choice(title="Residential", value="RESIDENTIAL"), + questionary.Choice(title="General", value="GENERAL"), + questionary.Choice(title="Special Use", value="SPECIAL_USE"), + ], + validate=lambda _: True if _ else "Select at least one customer class", + ).ask(), + ) + + +def _select_tariff_types() -> list[TariffType]: + return cast( + list[TariffType], + questionary.checkbox( + message="Select tariff types", + choices=[ + questionary.Choice(title="Default", value="DEFAULT"), + questionary.Choice(title="Alternative", value="ALTERNATIVE"), + questionary.Choice(title="Optional extra", value="OPTIONAL_EXTRA"), + questionary.Choice(title="Rider", value="RIDER"), + ], + validate=lambda _: bool(_) or "Select at least one tariff type", + ).ask(), + ) + + +def _fetch_tariffs(api: ArcadiaSignalAPI, tariffs: list[tuple[str, int]], year: int): + result: list[TariffExtended] = [] + with console.status("Fetching tariffs..."): + for name, id_ in tariffs: + console.print(f"Tariff id: {name}") + page = api.tariffs.iter_pages( + fields="ext", + master_tariff_id=id_, + effective_on=date(year, 12, 1), + populate_properties=True, + populate_rates=True, + ) + result.extend(page) + return result + + +def _prompt_tariff_name(default: str) -> str: + result = cast(str | None, questionary.text("Tariff name", default=default).ask()) + if result is None: + exit() + return result diff --git a/tariff_fetch/urdb/arcadia/build.py b/tariff_fetch/urdb/arcadia/build.py index 0a762cb..4ccbb97 100644 --- a/tariff_fetch/urdb/arcadia/build.py +++ b/tariff_fetch/urdb/arcadia/build.py @@ -9,6 +9,7 @@ from .energyschedule import build_energy_schedule from .fixedcharge import build_fixed_charge +from .metadata import build_metadata from .scenario import Scenario @@ -28,7 +29,13 @@ def build_urdb(api: ArcadiaSignalAPI, scenario: Scenario) -> URDBRate: fixed_charge = build_fixed_charge(scenario, library) except Exception as e: fixed_charge = _confirm_proceed(e, "fixed charges") - return {**energy_schedule, **fixed_charge} + + try: + metadata = build_metadata(scenario, library) + except Exception as e: + metadata = _confirm_proceed(e, "metadata") + + return {**energy_schedule, **fixed_charge, **metadata} def _confirm_proceed(e: Exception, processing: str) -> URDBRate: diff --git a/tariff_fetch/urdb/arcadia/fixedcharge.py b/tariff_fetch/urdb/arcadia/fixedcharge.py new file mode 100644 index 0000000..afe60b2 --- /dev/null +++ b/tariff_fetch/urdb/arcadia/fixedcharge.py @@ -0,0 +1,74 @@ +from collections.abc import Iterator +from datetime import datetime, timedelta +from statistics import mean + +from t import Library, Scenario +from tariff_fetch.arcadia.schema.tariffrate import TariffRateExtended +from tariff_fetch.urdb.arcadia.exception import RateConversionError +from tariff_fetch.urdb.schema import URDBRate + +from . import rateutils as ru + +_LOGGED: set[int] = set() + + +def build_fixed_charge(scenario: Scenario, library: Library) -> URDBRate: + return { + "fixedchargefirstmeter": get_fixed_charge_value(scenario, library), + "fixedchargeunits": "$/month", + } + + +def get_fixed_charge_value(scenario: Scenario, library: Library) -> float: + return mean(get_fixed_charge_at_dt(scenario, library, dt) for dt in _iter_year(scenario.year, timedelta(hours=12))) + + +def get_fixed_charge_at_dt(scenario: Scenario, library: Library, dt: datetime) -> float: + master_tariff_id = scenario.master_tariff_id + tariff = library.tariffs.get_tariff_at_date(master_tariff_id, dt.date()) + rates = ru.tariff_iter_rates_for_dt(tariff, scenario, library, dt) + return sum(get_rate_fixed_charge_at_dt(scenario, library, rate, dt) for rate in rates) + + +def get_rate_fixed_charge_at_dt(scenario: Scenario, library: Library, rate: TariffRateExtended, dt: datetime) -> float: + bands = ru.rate_filter_bands(rate, scenario, library) + band_rate_units = {band["rate_unit"] for band in bands} + if rate["charge_type"] != "FIXED_PRICE": + return 0 + if "COST_PER_UNIT" not in band_rate_units: + raise RateConversionError(rate, "Fixed price rate bands units should be COST_PER_UNIT") + if rate["charge_period"] != "MONTHLY": + raise RateConversionError(rate, "Fixed charges should be monthly") + if len(band_rate_units) > 1: + raise RateConversionError(rate, "More than one applicable band for percentage rate") + band = bands[0] + if band["has_consumption_limit"]: + raise RateConversionError(rate, "Fixed rate bands cannot have has_consumption_limit==true") + if band.get("consumption_upper_limit") is not None: + raise RateConversionError(rate, "Fixed rate bands cannot have consumption_upper_limit") + if band["has_demand_limit"]: + raise RateConversionError(rate, "Fixed rate bands cannot have has_demand_limit==true") + if band.get("demand_upper_limit") is not None: + raise RateConversionError(rate, "Fixed rate bands cannot have demand_upper_limit") + if band["has_property_limit"]: + raise RateConversionError(rate, "Fixed rate bands cannot have has_property_limit==true") + if band.get("property_upper_limit") is not None: + raise RateConversionError(rate, "Fixed rate bands cannot have property_upper_limit") + if band.get("calculation_factor") is not None: + raise RateConversionError(rate, "Fixed rate bands cannot have calculation_factor") + if band.get("applicability_formula") is not None: + raise RateConversionError(rate, "Fixed rate bands cannot have applicability_formula") + + rate_amount = ru.rate_band_get_amount_at_datetime(band, library, dt) + if rate["tariff_rate_id"] not in _LOGGED: + print(f"Applied fixed charge: {rate['rate_name']} ({rate['tariff_rate_id']}) ({rate_amount})") + _LOGGED.add(rate["tariff_rate_id"]) + return rate_amount + + +def _iter_year(year: int, delta: timedelta) -> Iterator[datetime]: + dt = datetime(year, 1, 1, 0, 30, 0) + max_dt = datetime(year + 1, 1, 1, 0, 0, 0) + while dt < max_dt: + yield dt + dt += delta diff --git a/tariff_fetch/urdb/arcadia/metadata.py b/tariff_fetch/urdb/arcadia/metadata.py new file mode 100644 index 0000000..7ca4d42 --- /dev/null +++ b/tariff_fetch/urdb/arcadia/metadata.py @@ -0,0 +1,12 @@ +from datetime import date + +from tariff_fetch.urdb.schema import URDBRate + +from .library import Library +from .scenario import Scenario + + +def build_metadata(scenario: Scenario, library: Library) -> URDBRate: + master_tariff_id = scenario.master_tariff_id + tariff = library.tariffs.get_tariff_at_date(master_tariff_id, date(scenario.year, 1, 1)) + return {"label": tariff["lse_code"], "utility": tariff["lse_name"], "name": tariff["tariff_name"], "country": "USA"} diff --git a/uv.lock b/uv.lock index 31a998b..a8f775b 100644 --- a/uv.lock +++ b/uv.lock @@ -1480,7 +1480,7 @@ wheels = [ [[package]] name = "tariff-fetch" -version = "0.3.0" +version = "0.4.0" source = { editable = "." } dependencies = [ { name = "fastexcel" }, From 4e41a787f330895f2929fc6919c3235c0f4e6810 Mon Sep 17 00:00:00 2001 From: Natalia Borovskikh Date: Wed, 4 Mar 2026 19:31:29 +0000 Subject: [PATCH 06/24] Fix for negative rates being ignored --- pyproject.toml | 2 +- tariff_fetch/urdb/rateacuity_history_gas/__init__.py | 2 ++ tariff_fetch/urdb/rateacuity_history_gas/history_data.py | 2 +- uv.lock | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6e83252..281424a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "tariff_fetch" -version = "0.4.0" +version = "0.4.1" description = "A CLI tool and a python library to simplify downloading electric and gas utility tariff data." authors = [{ name = "Switchbox", email = "hello@switch.box" }] readme = "README.md" diff --git a/tariff_fetch/urdb/rateacuity_history_gas/__init__.py b/tariff_fetch/urdb/rateacuity_history_gas/__init__.py index 6633b85..b680f12 100644 --- a/tariff_fetch/urdb/rateacuity_history_gas/__init__.py +++ b/tariff_fetch/urdb/rateacuity_history_gas/__init__.py @@ -48,6 +48,8 @@ def _build_energy_schedule_raw(rows: Collection[Row], include_taxes: bool) -> UR summed_bands[0], *(this for prev, this in itertools.pairwise(summed_bands) if this[0] - prev[0] > 30), ] + # clamp at >0 + summed_bands = [(bound, max(0, value)) for bound, value in summed_bands] month_bands.append(tuple(summed_bands)) month_bands_unique = list(set(month_bands)) energy_weekday_schedule = cast(MonthSchedule, tuple(tuple([month_bands_unique.index(b)] * 24) for b in month_bands)) diff --git a/tariff_fetch/urdb/rateacuity_history_gas/history_data.py b/tariff_fetch/urdb/rateacuity_history_gas/history_data.py index ed1005b..2693e8c 100644 --- a/tariff_fetch/urdb/rateacuity_history_gas/history_data.py +++ b/tariff_fetch/urdb/rateacuity_history_gas/history_data.py @@ -116,7 +116,7 @@ def month_value_kwh( raw_value = self.month_values[month] if raw_value is None: return 0 - result = max(0, raw_value / kwh_multiplier(self.rate_determinant)) + result = raw_value / kwh_multiplier(self.rate_determinant) if self.location: result *= self.location_avg_factor return result diff --git a/uv.lock b/uv.lock index a8f775b..f8ff12f 100644 --- a/uv.lock +++ b/uv.lock @@ -1480,7 +1480,7 @@ wheels = [ [[package]] name = "tariff-fetch" -version = "0.4.0" +version = "0.4.1" source = { editable = "." } dependencies = [ { name = "fastexcel" }, From 59e3f22225ec8e8be23204a3069330da1a56c34e Mon Sep 17 00:00:00 2001 From: Natalia Borovskikh Date: Thu, 5 Mar 2026 12:13:00 +0000 Subject: [PATCH 07/24] Preserve rate energy structure ordering --- pyproject.toml | 2 +- tariff_fetch/urdb/arcadia/library.py | 2 +- tariff_fetch/urdb/rateacuity_history_gas/__init__.py | 4 ++-- uv.lock | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 281424a..248c51a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "tariff_fetch" -version = "0.4.1" +version = "0.4.3" description = "A CLI tool and a python library to simplify downloading electric and gas utility tariff data." authors = [{ name = "Switchbox", email = "hello@switch.box" }] readme = "README.md" diff --git a/tariff_fetch/urdb/arcadia/library.py b/tariff_fetch/urdb/arcadia/library.py index ac0a3e8..fe6da3e 100644 --- a/tariff_fetch/urdb/arcadia/library.py +++ b/tariff_fetch/urdb/arcadia/library.py @@ -1,6 +1,5 @@ from datetime import date, datetime from typing import Literal, final, overload -from xdrlib import ConversionError from tariff_fetch.arcadia.api import ArcadiaSignalAPI from tariff_fetch.arcadia.schema.lookup import Lookup @@ -9,6 +8,7 @@ from tariff_fetch.arcadia.schema.tariffrate import TariffRateExtended from tariff_fetch.urdb.arcadia.exception import TariffNotFoundByDate, TariffNotFoundById +from .exception import ConversionError from .prompts import ( prompt_boolean, prompt_choice, diff --git a/tariff_fetch/urdb/rateacuity_history_gas/__init__.py b/tariff_fetch/urdb/rateacuity_history_gas/__init__.py index b680f12..e9e223e 100644 --- a/tariff_fetch/urdb/rateacuity_history_gas/__init__.py +++ b/tariff_fetch/urdb/rateacuity_history_gas/__init__.py @@ -51,8 +51,8 @@ def _build_energy_schedule_raw(rows: Collection[Row], include_taxes: bool) -> UR # clamp at >0 summed_bands = [(bound, max(0, value)) for bound, value in summed_bands] month_bands.append(tuple(summed_bands)) - month_bands_unique = list(set(month_bands)) - energy_weekday_schedule = cast(MonthSchedule, tuple(tuple([month_bands_unique.index(b)] * 24) for b in month_bands)) + month_bands_unique = list(dict.fromkeys(month_bands)) + energy_weekday_schedule = tuple(tuple([month_bands_unique.index(b)] * 24) for b in month_bands) energy_weekend_schedule = energy_weekday_schedule energy_rate_structure = [[_band_tuple_to_tier(br) for br in mb if br != (0, 0)] for mb in month_bands_unique] return { diff --git a/uv.lock b/uv.lock index f8ff12f..6393b3f 100644 --- a/uv.lock +++ b/uv.lock @@ -1480,7 +1480,7 @@ wheels = [ [[package]] name = "tariff-fetch" -version = "0.4.1" +version = "0.4.3" source = { editable = "." } dependencies = [ { name = "fastexcel" }, From baec3023d8eca91fb472cbf75166ee7f9ee55496 Mon Sep 17 00:00:00 2001 From: Natalia Borovskikh Date: Thu, 5 Mar 2026 12:14:08 +0000 Subject: [PATCH 08/24] Remove unused imports --- tariff_fetch/urdb/rateacuity_history_gas/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tariff_fetch/urdb/rateacuity_history_gas/__init__.py b/tariff_fetch/urdb/rateacuity_history_gas/__init__.py index e9e223e..0e64a40 100644 --- a/tariff_fetch/urdb/rateacuity_history_gas/__init__.py +++ b/tariff_fetch/urdb/rateacuity_history_gas/__init__.py @@ -1,9 +1,8 @@ import itertools from collections.abc import Collection from math import inf -from typing import cast -from tariff_fetch.urdb.schema import EnergyTier, MonthSchedule, URDBRate +from tariff_fetch.urdb.schema import EnergyTier, URDBRate from .exceptions import EmptyBandsError from .history_data import ConsumptionRow, FixedChargeRow, PercentageRow, Row From be6dd7d2ca22c67fa8fccc3aa0a28305c73234b9 Mon Sep 17 00:00:00 2001 From: Natalia Borovskikh Date: Thu, 12 Mar 2026 12:12:50 +0000 Subject: [PATCH 09/24] A couple fixes --- pyproject.toml | 2 +- tariff_fetch/_cli/arcadia_urdb.py | 3 +-- tariff_fetch/arcadia/schema/common.py | 9 ++++++--- tariff_fetch/arcadia/schema/tariffproperty.py | 1 + tariff_fetch/arcadia/schema/tariffrate.py | 4 ++++ tariff_fetch/urdb/arcadia/energyschedule.py | 16 ++++++++++++++-- tariff_fetch/urdb/arcadia/fixedcharge.py | 9 +++++++-- tariff_fetch/urdb/arcadia/library.py | 2 ++ uv.lock | 2 +- 9 files changed, 37 insertions(+), 11 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 248c51a..3a5a3ff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "tariff_fetch" -version = "0.4.3" +version = "0.4.5" description = "A CLI tool and a python library to simplify downloading electric and gas utility tariff data." authors = [{ name = "Switchbox", email = "hello@switch.box" }] readme = "README.md" diff --git a/tariff_fetch/_cli/arcadia_urdb.py b/tariff_fetch/_cli/arcadia_urdb.py index 3cdf7a4..e3a2d5c 100644 --- a/tariff_fetch/_cli/arcadia_urdb.py +++ b/tariff_fetch/_cli/arcadia_urdb.py @@ -2,11 +2,10 @@ import os from datetime import date from pathlib import Path -from typing import cast, get_args +from typing import cast import questionary from dotenv import load_dotenv -from pydantic import TypeAdapter # from tariff_fetch.genability.lse import get_lses_page # from tariff_fetch.genability.tariffs import CustomerClass, TariffType, tariffs_paginate diff --git a/tariff_fetch/arcadia/schema/common.py b/tariff_fetch/arcadia/schema/common.py index a3bd772..5386ffc 100644 --- a/tariff_fetch/arcadia/schema/common.py +++ b/tariff_fetch/arcadia/schema/common.py @@ -17,7 +17,7 @@ "MAXIMUM", "TAX", ] -TariffChargePeriod = Literal["MONTHLY", "DAILY", "QUARTERLY", "ANNUALLY", "HOURLY"] +TariffChargePeriod = Literal["MONTHLY", "DAILY", "QUARTERLY", "ANNUALLY", "HOURLY", "ONE_TIME"] RateChargeClass = Literal[ "SUPPLY", @@ -29,8 +29,9 @@ "AFTER_TAX", "OTHER", "NON_BYPASSABLE", + "NET_EXCESS", ] -RateTransactionType = Literal["BUY", "SELL", "NET", "BUY_IMPORT", "SELL_IMPORT"] +RateTransactionType = Literal["BUY", "SELL", "NET", "BUY_IMPORT", "SELL_IMPORT", "SELL_EXPORT"] RateUnit = Literal["COST_PER_UNIT", "PERCENTAGE"] TerritoryUsageType = Literal["SERVICE", "TARIFF", "CLIMATE_ZONE", "UTILITY_CLIMATE_ZONE"] @@ -38,5 +39,7 @@ SeasonPredominance = Literal["PREDOMINANT", "SUBSERVIENT"] -TimeOfUseType = Literal["SUPER_OFF_PEAK", "OFF_PEAK", "PARTIAL_PEAK", "ON_PEAK", "SUPER_ON_PEAK", "CRITICAL_PEAK"] +TimeOfUseType = Literal[ + "SUPER_OFF_PEAK", "OFF_PEAK", "PARTIAL_PEAK", "ON_PEAK", "SUPER_ON_PEAK", "CRITICAL_PEAK", "ALL" +] TimeOfUsePrivacy = Literal["PUBLIC", "PRIVATE"] diff --git a/tariff_fetch/arcadia/schema/tariffproperty.py b/tariff_fetch/arcadia/schema/tariffproperty.py index 02f79b4..9ba22b2 100644 --- a/tariff_fetch/arcadia/schema/tariffproperty.py +++ b/tariff_fetch/arcadia/schema/tariffproperty.py @@ -35,6 +35,7 @@ "OFF_PEAK", "CRITICAL_PEAK", "SUPER_ON_PEAK", + "ALL", # Undocumented ] diff --git a/tariff_fetch/arcadia/schema/tariffrate.py b/tariff_fetch/arcadia/schema/tariffrate.py index 0febe1a..f54cb3f 100644 --- a/tariff_fetch/arcadia/schema/tariffrate.py +++ b/tariff_fetch/arcadia/schema/tariffrate.py @@ -129,6 +129,10 @@ class TariffRateStandardFields(TypedDict): """ rate_bands: list[TariffRateBand] + # These are fields with unknown purpose + edge_predominance: NotRequired[str] + proration_rules: NotRequired[list[str]] + class TariffRateExtendedFields(TypedDict): rider_tariff_id: NotRequired[int] diff --git a/tariff_fetch/urdb/arcadia/energyschedule.py b/tariff_fetch/urdb/arcadia/energyschedule.py index da79d0b..d01166f 100644 --- a/tariff_fetch/urdb/arcadia/energyschedule.py +++ b/tariff_fetch/urdb/arcadia/energyschedule.py @@ -23,11 +23,11 @@ def build_energy_schedule(scenario: Scenario, library: Library) -> URDBRate: - weekend_schedule_raw = [ + weekday_schedule_raw = [ [get_month_hour_bands(scenario, library, month, hour, is_weekday) for hour in range(24)] for month in range(1, 13) ] - weekday_schedule_raw = [ + weekend_schedule_raw = [ [get_month_hour_bands(scenario, library, month, hour, is_weekend) for hour in range(24)] for month in range(1, 13) ] @@ -99,6 +99,11 @@ def _get_rate_consumption_bands_at_datetime( ) -> ConsumptionBandSet | None: if rate.get("charge_type") != "CONSUMPTION_BASED": return None + if (transaction_type := rate["transaction_type"]) not in {"BUY", "NET", "BUY_IMPORT"}: + raise RateConversionError( + rate, + f"Only BUY, BUY_IMPORT, and NET transactions are supported for consumption rates (got {transaction_type})", + ) if set(rate.get("charge_class", [])) - _ALLOWED_CHARGE_CLASSES: raise RateConversionError(rate, "Incorrect charge class for consumption-based rate") if rate["charge_period"] != "MONTHLY": @@ -121,6 +126,13 @@ def _get_percentage_rates_at_datetime( result: PercentageModifiers = [] for rate in rates: + if (transaction_type := rate["transaction_type"]) not in {"BUY", "NET", "BUY_IMPORT"}: + raise RateConversionError( + rate, + f"Only BUY, BUY_IMPORT, and NET transactions are supported for percentage rates (got {transaction_type})", + ) + if rate["charge_period"] != "MONTHLY": + raise RateConversionError(rate, "Incorrect charge period for percentage-based rate") if not (bands := ru.rate_filter_bands(rate, scenario, library)): raise RateConversionError(rate, "No bands for percentage rate") diff --git a/tariff_fetch/urdb/arcadia/fixedcharge.py b/tariff_fetch/urdb/arcadia/fixedcharge.py index afe60b2..a1360fc 100644 --- a/tariff_fetch/urdb/arcadia/fixedcharge.py +++ b/tariff_fetch/urdb/arcadia/fixedcharge.py @@ -2,12 +2,13 @@ from datetime import datetime, timedelta from statistics import mean -from t import Library, Scenario from tariff_fetch.arcadia.schema.tariffrate import TariffRateExtended -from tariff_fetch.urdb.arcadia.exception import RateConversionError from tariff_fetch.urdb.schema import URDBRate from . import rateutils as ru +from .exception import RateConversionError +from .library import Library +from .scenario import Scenario _LOGGED: set[int] = set() @@ -35,6 +36,10 @@ def get_rate_fixed_charge_at_dt(scenario: Scenario, library: Library, rate: Tari band_rate_units = {band["rate_unit"] for band in bands} if rate["charge_type"] != "FIXED_PRICE": return 0 + if (transaction_type := rate["transaction_type"]) != "BUY": + raise RateConversionError( + rate, f"Only BUY transaction type is supported for fixed charges (got {transaction_type})" + ) if "COST_PER_UNIT" not in band_rate_units: raise RateConversionError(rate, "Fixed price rate bands units should be COST_PER_UNIT") if rate["charge_period"] != "MONTHLY": diff --git a/tariff_fetch/urdb/arcadia/library.py b/tariff_fetch/urdb/arcadia/library.py index fe6da3e..4e23d2e 100644 --- a/tariff_fetch/urdb/arcadia/library.py +++ b/tariff_fetch/urdb/arcadia/library.py @@ -68,6 +68,8 @@ def _fetch_tariff(self, tariff_id: int) -> TariffExtended: fields="ext", search=str(tariff_id), search_on=["tariffId"], + starts_with=True, + ends_with=True, populate_properties=True, populate_rates=True, ) diff --git a/uv.lock b/uv.lock index 6393b3f..622a719 100644 --- a/uv.lock +++ b/uv.lock @@ -1480,7 +1480,7 @@ wheels = [ [[package]] name = "tariff-fetch" -version = "0.4.3" +version = "0.4.5" source = { editable = "." } dependencies = [ { name = "fastexcel" }, From 33e024baa79bf45f416f84ec591a8122da512beb Mon Sep 17 00:00:00 2001 From: Natalia Borovskikh Date: Thu, 12 Mar 2026 12:55:38 +0000 Subject: [PATCH 10/24] add tests --- .gitignore | 2 + pyproject.toml | 23 ++- tariff_fetch/_cli/arcadia_urdb.py | 2 +- tariff_fetch/_cli/genability.py | 2 +- tariff_fetch/urdb/arcadia/energyschedule.py | 12 +- tariff_fetch/urdb/arcadia/exception.py | 2 +- tariff_fetch/urdb/arcadia/fixedcharge.py | 8 +- tariff_fetch/urdb/arcadia/library.py | 65 +++++- tariff_fetch/urdb/arcadia/prompts.py | 3 +- tariff_fetch/urdb/arcadia/rateutils.py | 2 +- tariff_fetch/urdb/arcadia/scenario.py | 6 +- tests/test_arcadia_urdb_boundaries_build.py | 119 +++++++++++ tests/test_arcadia_urdb_energy.py | 87 ++++++++ tests/test_arcadia_urdb_fixedcharge.py | 27 +++ tests/test_arcadia_urdb_golden.py | 94 +++++++++ tests/test_arcadia_urdb_library_debug.py | 39 ++++ tests/test_arcadia_urdb_rateutils.py | 216 ++++++++++++++++++++ tests/test_rateacuity_urdb.py | 1 + 18 files changed, 682 insertions(+), 28 deletions(-) create mode 100644 tests/test_arcadia_urdb_boundaries_build.py create mode 100644 tests/test_arcadia_urdb_energy.py create mode 100644 tests/test_arcadia_urdb_fixedcharge.py create mode 100644 tests/test_arcadia_urdb_golden.py create mode 100644 tests/test_arcadia_urdb_library_debug.py create mode 100644 tests/test_arcadia_urdb_rateutils.py create mode 100644 tests/test_rateacuity_urdb.py diff --git a/.gitignore b/.gitignore index e28aee6..efd32f8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +outputs + # Python __pycache__/ *.py[cod] diff --git a/pyproject.toml b/pyproject.toml index 3a5a3ff..62312e7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,5 +76,26 @@ ignore = ["E501", "SIM103"] "tests/*" = ["S101"] [tool.basedpyright] -include = ["tariff_fetch"] +include = ["tariff_fetch", "tests"] pythonVersion = "3.11" +venvPath = "." +venv = ".venv" + +[[tool.basedpyright.executionEnvironments]] +root = "tests" +extraPaths = [".."] +reportUnknownVariableType = "none" +reportUnknownMemberType = "none" +reportUnknownParameterType = "none" +reportUnknownLambdaType = "none" +reportUnknownArgumentType = "none" +reportArgumentType = "none" +reportMissingParameterType = "none" +reportMissingTypeArgument = "none" +reportMissingTypeStubs = "none" +reportUnusedParameter = "none" +reportUnusedImport = "none" +reportUnusedCallResult = "none" +reportUnannotatedClassAttribute = "none" +reportUnnecessaryTypeIgnoreComment = "none" +reportPrivateUsage = "none" diff --git a/tariff_fetch/_cli/arcadia_urdb.py b/tariff_fetch/_cli/arcadia_urdb.py index e3a2d5c..86652c7 100644 --- a/tariff_fetch/_cli/arcadia_urdb.py +++ b/tariff_fetch/_cli/arcadia_urdb.py @@ -10,7 +10,7 @@ # from tariff_fetch.genability.lse import get_lses_page # from tariff_fetch.genability.tariffs import CustomerClass, TariffType, tariffs_paginate from tariff_fetch.arcadia.api import ArcadiaSignalAPI -from tariff_fetch.arcadia.schema.common import CustomerClass, RateChargeClass, TariffType +from tariff_fetch.arcadia.schema.common import CustomerClass, TariffType from tariff_fetch.arcadia.schema.tariff import TariffExtended from tariff_fetch.urdb.arcadia.build import build_urdb from tariff_fetch.urdb.arcadia.prompts import prompt_charge_classes diff --git a/tariff_fetch/_cli/genability.py b/tariff_fetch/_cli/genability.py index 680e4e6..360b138 100644 --- a/tariff_fetch/_cli/genability.py +++ b/tariff_fetch/_cli/genability.py @@ -1,5 +1,5 @@ import os -from datetime import date, datetime +from datetime import date from pathlib import Path from typing import cast diff --git a/tariff_fetch/urdb/arcadia/energyschedule.py b/tariff_fetch/urdb/arcadia/energyschedule.py index d01166f..6697b6a 100644 --- a/tariff_fetch/urdb/arcadia/energyschedule.py +++ b/tariff_fetch/urdb/arcadia/energyschedule.py @@ -75,10 +75,10 @@ def get_raw_bands_at_datetime(scenario: Scenario, library: Library, dt: datetime """Get raw tariff consumption-based bands at datetime dt""" tariff = library.tariffs.get_tariff_at_date(scenario.master_tariff_id, dt) rates = list(ru.tariff_iter_rates_for_dt(tariff, scenario, library, dt)) - percentage_modifiers = _get_percentage_rates_at_datetime(rates, scenario, library, dt) + percentage_modifiers = get_percentage_rates_at_datetime(rates, scenario, library, dt) piecewise_bands: list[ConsumptionBandSet] = [] for rate in rates: - rate_bands = _get_rate_consumption_bands_at_datetime(rate, scenario, library, dt) + rate_bands = get_rate_consumption_bands_at_datetime(rate, scenario, library, dt) if rate_bands is None: continue # Apply percentage modifiers @@ -88,10 +88,10 @@ def get_raw_bands_at_datetime(scenario: Scenario, library: Library, dt: datetime rate_bands = [(limit, value * charge_multiplier) for limit, value in rate_bands] piecewise_bands.append(rate_bands) - return _sum_piecewise_bands(piecewise_bands) + return sum_piecewise_bands(piecewise_bands) -def _get_rate_consumption_bands_at_datetime( +def get_rate_consumption_bands_at_datetime( rate: TariffRateExtended, scenario: Scenario, library: Library, @@ -119,7 +119,7 @@ def _get_rate_consumption_bands_at_datetime( ] -def _get_percentage_rates_at_datetime( +def get_percentage_rates_at_datetime( rates: Collection[TariffRateExtended], scenario: Scenario, library: Library, dt: datetime ) -> PercentageModifiers: rates = [rate for rate in rates if ru.rate_get_band_units(rate) == {"PERCENTAGE"}] @@ -151,7 +151,7 @@ def _get_percentage_rates_at_datetime( return result -def _sum_piecewise_bands(inputs: Collection[ConsumptionBandSet]) -> ConsumptionBandSet: +def sum_piecewise_bands(inputs: Collection[ConsumptionBandSet]) -> ConsumptionBandSet: if not inputs: return [] diff --git a/tariff_fetch/urdb/arcadia/exception.py b/tariff_fetch/urdb/arcadia/exception.py index 04eff99..72095b0 100644 --- a/tariff_fetch/urdb/arcadia/exception.py +++ b/tariff_fetch/urdb/arcadia/exception.py @@ -1,8 +1,8 @@ from datetime import date from typing import final + from typing_extensions import override -from tariff_fetch.arcadia.schema.tariff import TariffExtended from tariff_fetch.arcadia.schema.tariffrate import TariffRateExtended diff --git a/tariff_fetch/urdb/arcadia/fixedcharge.py b/tariff_fetch/urdb/arcadia/fixedcharge.py index a1360fc..0813fef 100644 --- a/tariff_fetch/urdb/arcadia/fixedcharge.py +++ b/tariff_fetch/urdb/arcadia/fixedcharge.py @@ -33,17 +33,19 @@ def get_fixed_charge_at_dt(scenario: Scenario, library: Library, dt: datetime) - def get_rate_fixed_charge_at_dt(scenario: Scenario, library: Library, rate: TariffRateExtended, dt: datetime) -> float: bands = ru.rate_filter_bands(rate, scenario, library) - band_rate_units = {band["rate_unit"] for band in bands} if rate["charge_type"] != "FIXED_PRICE": return 0 + if not bands: + return 0 + band_rate_units = {band["rate_unit"] for band in bands} if (transaction_type := rate["transaction_type"]) != "BUY": raise RateConversionError( rate, f"Only BUY transaction type is supported for fixed charges (got {transaction_type})" ) if "COST_PER_UNIT" not in band_rate_units: raise RateConversionError(rate, "Fixed price rate bands units should be COST_PER_UNIT") - if rate["charge_period"] != "MONTHLY": - raise RateConversionError(rate, "Fixed charges should be monthly") + if (charge_period := rate["charge_period"]) != "MONTHLY": + raise RateConversionError(rate, f"Fixed charges should be monthly (got {charge_period})") if len(band_rate_units) > 1: raise RateConversionError(rate, "More than one applicable band for percentage rate") band = bands[0] diff --git a/tariff_fetch/urdb/arcadia/library.py b/tariff_fetch/urdb/arcadia/library.py index 4e23d2e..5ef0294 100644 --- a/tariff_fetch/urdb/arcadia/library.py +++ b/tariff_fetch/urdb/arcadia/library.py @@ -1,4 +1,6 @@ +import json from datetime import date, datetime +from pathlib import Path from typing import Literal, final, overload from tariff_fetch.arcadia.api import ArcadiaSignalAPI @@ -20,12 +22,48 @@ ) PropertyValue = str | list[str] | bool | date | float | int +_DEFAULT_DEBUG_ROOT = Path("./outputs/arcadia_library") + + +def _json_default(value: object) -> str: + if isinstance(value, datetime | date): + return value.isoformat() + raise TypeError(f"Object of type {type(value).__name__} is not JSON serializable") + + +@final +class LibraryDebugStore: + def __init__(self, root: Path = _DEFAULT_DEBUG_ROOT) -> None: + self.root = root + self.tariffs_dir = root / "tariffs" + self.lookups_dir = root / "lookups" + self.properties_dir = root / "properties" + + def save_tariff(self, tariff: TariffExtended) -> None: + effective_date = tariff["effective_date"].isoformat() + tariff_id = tariff["tariff_id"] + master_tariff_id = tariff["master_tariff_id"] + path = self.tariffs_dir / f"master-{master_tariff_id}_tariff-{tariff_id}_effective-{effective_date}.json" + self._write_json(path, tariff) + + def save_lookups(self, key: str, year: int, lookups: list[Lookup]) -> None: + path = self.lookups_dir / f"{key}_{year}.json" + self._write_json(path, lookups) + + def save_property_value(self, key: str, value: PropertyValue) -> None: + path = self.properties_dir / f"{key}.json" + self._write_json(path, {"key": key, "value": value}) + + def _write_json(self, path: Path, payload: object) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(json.dumps(payload, indent=2, default=_json_default)) @final class TariffLibrary: - def __init__(self, api: ArcadiaSignalAPI) -> None: + def __init__(self, api: ArcadiaSignalAPI, debug_store: LibraryDebugStore) -> None: self.api = api + self.debug_store = debug_store self.tariffs: list[TariffExtended] = [] def get_tariff_at_date(self, master_tariff_id: int, dt: date) -> TariffExtended: @@ -33,14 +71,14 @@ def get_tariff_at_date(self, master_tariff_id: int, dt: date) -> TariffExtended: if (found := self._find_tariff_at_date(master_tariff_id, dt)) is not None: return found tariff = self._fetch_tariff_at_date(master_tariff_id, dt) - self.tariffs.append(tariff) + self._remember(tariff) return tariff def get_tariff(self, tariff_id: int) -> TariffExtended: if (found := self._find_tariff(tariff_id)) is not None: return found tariff = self._fetch_tariff(tariff_id) - self.tariffs.append(tariff) + self._remember(tariff) return tariff def get_rate(self, rate_id: int) -> TariffRateExtended: @@ -62,6 +100,10 @@ def _find_tariff_at_date(self, master_tariff_id: int, dt: date) -> TariffExtende return tariff return None + def _remember(self, tariff: TariffExtended) -> None: + self.tariffs.append(tariff) + self.debug_store.save_tariff(tariff) + def _fetch_tariff(self, tariff_id: int) -> TariffExtended: tariffs = list( self.api.tariffs.iter_pages( @@ -96,14 +138,16 @@ def _fetch_tariff_at_date(self, master_tariff_id: int, dt: date) -> TariffExtend @final class VariablePropertyLibrary: - def __init__(self, api: ArcadiaSignalAPI): + def __init__(self, api: ArcadiaSignalAPI, debug_store: LibraryDebugStore): self.api = api + self.debug_store = debug_store self.property_timeseries: dict[tuple[str, int], list[Lookup]] = {} def lookup(self, key: str, dt: datetime) -> float: if (lookups := self.property_timeseries.get((key, dt.year))) is None: lookups = self._lookup_property_timeseries(key, dt.year) self.property_timeseries[(key, dt.year)] = lookups + self.debug_store.save_lookups(key, dt.year, lookups) for row in lookups: if row["from_date_time"] <= dt <= (row["to_date_time"] or datetime.max): @@ -128,10 +172,16 @@ def _lookup_property_timeseries(self, key: str, year: int) -> list[Lookup]: @final class Library: - def __init__(self, api: ArcadiaSignalAPI, properties: dict[str, PropertyValue] | None = None): + def __init__( + self, + api: ArcadiaSignalAPI, + properties: dict[str, PropertyValue] | None = None, + debug_root: Path = _DEFAULT_DEBUG_ROOT, + ): self.api = api - self.tariffs = TariffLibrary(api) - self.variables = VariablePropertyLibrary(api) + self.debug_store = LibraryDebugStore(debug_root) + self.tariffs = TariffLibrary(api, self.debug_store) + self.variables = VariablePropertyLibrary(api, self.debug_store) self._properies: dict[str, PropertyValue] = properties or {} def has_property(self, key: str) -> bool: @@ -169,6 +219,7 @@ def get_property(self, key: str, data_type: TariffPropertyPrunedDataType) -> Pro if result is None: raise ConversionError("Property not set") self._properies[key] = result + self.debug_store.save_property_value(key, result) return result @overload diff --git a/tariff_fetch/urdb/arcadia/prompts.py b/tariff_fetch/urdb/arcadia/prompts.py index b08bad3..c25ad71 100644 --- a/tariff_fetch/urdb/arcadia/prompts.py +++ b/tariff_fetch/urdb/arcadia/prompts.py @@ -6,7 +6,6 @@ from tariff_fetch.arcadia.schema.common import RateChargeClass from tariff_fetch.arcadia.schema.tariffproperty import TariffPropertyStandard - # def prompt_scenario(master_tariff_id: int, year: int, apply_percentages: bool) -> Scenario | None: # result_charge_classes = prompt_charge_classes() # if result_charge_classes is None: @@ -99,7 +98,7 @@ def prompt_boolean(tariff_property: TariffPropertyStandard) -> bool | None: return result -def prompt_date(tariff_property: TariffPropertyStandard) -> date | None: +def prompt_date(tariff_property: TariffPropertyStandard) -> date | None: # pyright: ignore[reportUnusedParameter] raise NotImplementedError() diff --git a/tariff_fetch/urdb/arcadia/rateutils.py b/tariff_fetch/urdb/arcadia/rateutils.py index d13a07e..1dcd35c 100644 --- a/tariff_fetch/urdb/arcadia/rateutils.py +++ b/tariff_fetch/urdb/arcadia/rateutils.py @@ -71,7 +71,7 @@ def rate_is_applied_to_datetime(rate: TariffRateExtended, dt: datetime) -> bool: return True -def rate_filter_bands(rate: TariffRateExtended, scenario: Scenario, library: Library) -> list[TariffRateBand]: +def rate_filter_bands(rate: TariffRateExtended, _scenario: Scenario, library: Library) -> list[TariffRateBand]: result: list[TariffRateBand] = [] for band in rate.get("rate_bands"): if band.get("has_demand_limit"): diff --git a/tariff_fetch/urdb/arcadia/scenario.py b/tariff_fetch/urdb/arcadia/scenario.py index 8d89e69..b0848b4 100644 --- a/tariff_fetch/urdb/arcadia/scenario.py +++ b/tariff_fetch/urdb/arcadia/scenario.py @@ -1,11 +1,7 @@ from dataclasses import dataclass, field -from datetime import date -from typing import Literal, cast, get_args, overload +from typing import cast, get_args from tariff_fetch.arcadia.schema.common import RateChargeClass -from tariff_fetch.arcadia.schema.tariff import TariffExtended -from tariff_fetch.arcadia.schema.tariffproperty import TariffPropertyPrunedDataType -from tariff_fetch.urdb.arcadia.exception import TariffConversionError _CHARGE_CLASSES = cast(tuple[RateChargeClass, ...], get_args(RateChargeClass)) diff --git a/tests/test_arcadia_urdb_boundaries_build.py b/tests/test_arcadia_urdb_boundaries_build.py new file mode 100644 index 0000000..5daae78 --- /dev/null +++ b/tests/test_arcadia_urdb_boundaries_build.py @@ -0,0 +1,119 @@ +from datetime import date, datetime +from types import SimpleNamespace + +from tariff_fetch.urdb.arcadia import build as build_mod +from tariff_fetch.urdb.arcadia import metadata as metadata_mod +from tariff_fetch.urdb.arcadia import rateutils as ru +from tariff_fetch.urdb.arcadia.scenario import Scenario + + +def test_season_is_datetime_within_respects_start_inclusive_end_exclusive(): + season = { + "season_from_month": 6, + "season_from_day": 1, + "season_to_month": 9, + "season_to_day": 1, + } + + assert ru.season_is_datetime_within(season, date(2025, 6, 1)) is True # type: ignore[arg-type] + assert ru.season_is_datetime_within(season, date(2025, 8, 31)) is True # type: ignore[arg-type] + assert ru.season_is_datetime_within(season, date(2025, 9, 1)) is False # type: ignore[arg-type] + + +def test_season_is_datetime_within_handles_wraparound_year(): + season = { + "season_from_month": 11, + "season_from_day": 1, + "season_to_month": 3, + "season_to_day": 1, + } + + assert ru.season_is_datetime_within(season, date(2025, 12, 15)) is True # type: ignore[arg-type] + assert ru.season_is_datetime_within(season, date(2025, 2, 15)) is True # type: ignore[arg-type] + assert ru.season_is_datetime_within(season, date(2025, 5, 1)) is False # type: ignore[arg-type] + + +def test_period_is_datetime_within_respects_inclusive_boundaries(): + period = { + "from_day_of_week": 0, + "to_day_of_week": 4, + "from_hour": 9, + "to_hour": 17, + "from_minute": 0, + "to_minute": 59, + } + + assert ru.period_is_datetime_within(period, datetime(2025, 1, 6, 9, 0)) is True # type: ignore[arg-type] + assert ru.period_is_datetime_within(period, datetime(2025, 1, 10, 17, 59)) is True # type: ignore[arg-type] + assert ru.period_is_datetime_within(period, datetime(2025, 1, 11, 12, 0)) is False # type: ignore[arg-type] + assert ru.period_is_datetime_within(period, datetime(2025, 1, 6, 8, 59)) is False # type: ignore[arg-type] + + +def test_time_of_use_is_datetime_within_applies_embedded_season(): + tou = { + "season": { + "season_from_month": 6, + "season_from_day": 1, + "season_to_month": 9, + "season_to_day": 1, + }, + "tou_periods": [ + { + "from_day_of_week": 0, + "to_day_of_week": 6, + "from_hour": 14, + "to_hour": 18, + "from_minute": 0, + "to_minute": 59, + } + ], + } + + assert ru.time_of_use_is_datetime_within(tou, datetime(2025, 7, 10, 15, 30)) is True # type: ignore[arg-type] + assert ru.time_of_use_is_datetime_within(tou, datetime(2025, 10, 10, 15, 30)) is False # type: ignore[arg-type] + assert ru.time_of_use_is_datetime_within(tou, datetime(2025, 7, 10, 12, 30)) is False # type: ignore[arg-type] + + +def test_build_urdb_merges_converter_chunks(monkeypatch): + scenario = Scenario(123, 2025, apply_percentages=True, charge_classes={"SUPPLY"}) + + monkeypatch.setattr(build_mod, "Library", lambda api: SimpleNamespace()) + monkeypatch.setattr(build_mod, "build_energy_schedule", lambda scenario, library: {"energyratestructure": [[{"rate": 1.0, "unit": "kWh"}]]}) + monkeypatch.setattr(build_mod, "build_fixed_charge", lambda scenario, library: {"fixedchargefirstmeter": 12.0, "fixedchargeunits": "$/month"}) + monkeypatch.setattr(build_mod, "build_metadata", lambda scenario, library: {"label": "UTIL", "utility": "Utility", "name": "Tariff", "country": "USA"}) + + result = build_mod.build_urdb(api=object(), scenario=scenario) # type: ignore[arg-type] + + assert result == { + "energyratestructure": [[{"rate": 1.0, "unit": "kWh"}]], + "fixedchargefirstmeter": 12.0, + "fixedchargeunits": "$/month", + "label": "UTIL", + "utility": "Utility", + "name": "Tariff", + "country": "USA", + } + + +def test_build_metadata_reads_tariff_for_start_of_year(): + captured: list[tuple[int, date]] = [] + + def get_tariff_at_date(master_tariff_id: int, dt: date): + captured.append((master_tariff_id, dt)) + return { + "lse_code": "UTIL", + "lse_name": "Utility Name", + "tariff_name": "Residential Service", + } + + library = SimpleNamespace(tariffs=SimpleNamespace(get_tariff_at_date=get_tariff_at_date)) + + result = metadata_mod.build_metadata(Scenario(456, 2025, apply_percentages=False), library) # type: ignore[arg-type] + + assert captured == [(456, date(2025, 1, 1))] + assert result == { + "label": "UTIL", + "utility": "Utility Name", + "name": "Residential Service", + "country": "USA", + } diff --git a/tests/test_arcadia_urdb_energy.py b/tests/test_arcadia_urdb_energy.py new file mode 100644 index 0000000..7f92bb0 --- /dev/null +++ b/tests/test_arcadia_urdb_energy.py @@ -0,0 +1,87 @@ +from datetime import datetime +from math import inf + +import pytest + +from tariff_fetch.urdb.arcadia import energyschedule as es +from tariff_fetch.urdb.arcadia.exception import RateConversionError +from tariff_fetch.urdb.arcadia.shared import is_weekday + + +def test_build_energy_schedule_keeps_weekday_and_weekend_distinct(monkeypatch): + def fake_get_month_hour_bands(scenario, library, month, hour, day_filter): + if day_filter is is_weekday: + return [(inf, 1.0)] + return [(inf, 2.0)] + + monkeypatch.setattr(es, "get_month_hour_bands", fake_get_month_hour_bands) + + result = es.build_energy_schedule(scenario=None, library=None) # type: ignore[arg-type] + + assert result.get("energyratestructure") == [ + [{"rate": 1.0, "unit": "kWh"}], + [{"rate": 2.0, "unit": "kWh"}], + ] + assert "energyweekdayschedule" in result + assert "energyweekendschedule" in result + assert result["energyweekdayschedule"][0][0] == 0 + assert result["energyweekendschedule"][0][0] == 1 + + +def test_sum_piecewise_bands_aligns_and_collapses_values(): + result = es.sum_piecewise_bands( + [ + [(100.0, 1.0), (inf, 2.0)], + [(200.0, 0.5), (inf, 1.5)], + ] + ) + + assert result == [(100.0, 1.5), (200.0, 2.5), (inf, 3.5)] + + +def test_average_aligned_bands_averages_inputs_and_collapses_duplicates(): + result = es.average_aligned_bands( + [ + [(100.0, 1.0), (inf, 3.0)], + [(100.0, 3.0), (inf, 3.0)], + ] + ) + + assert result == [(100.0, 2.0), (inf, 3.0)] + + +def test_consumption_rate_rejects_unsupported_transaction_type(): + rate = { + "charge_type": "CONSUMPTION_BASED", + "transaction_type": "SELL_EXPORT", + "charge_class": ["SUPPLY"], + "charge_period": "MONTHLY", + "rate_bands": [], + } + + with pytest.raises(RateConversionError, match="Only BUY, BUY_IMPORT, and NET transactions are supported"): + es.get_rate_consumption_bands_at_datetime( + rate=rate, # pyright: ignore[reportArgumentType] + scenario=None, # pyright: ignore[reportArgumentType] + library=None, # pyright: ignore[reportArgumentType] + dt=datetime(2025, 1, 1, 0, 30), + ) + + +def test_percentage_rate_rejects_unsupported_transaction_type(monkeypatch): + rate = { + "transaction_type": "SELL_EXPORT", + "charge_period": "MONTHLY", + "charge_class": ["SUPPLY"], + "rate_bands": [{"rate_unit": "PERCENTAGE"}], + } + + monkeypatch.setattr(es.ru, "rate_get_band_units", lambda rate: {"PERCENTAGE"}) + + with pytest.raises(RateConversionError, match="Only BUY, BUY_IMPORT, and NET transactions are supported"): + es.get_percentage_rates_at_datetime( + rates=[rate], # pyright: ignore[reportArgumentType] + scenario=None, # pyright: ignore[reportArgumentType] + library=None, # pyright: ignore[reportArgumentType] + dt=datetime(2025, 1, 1, 0, 30), + ) diff --git a/tests/test_arcadia_urdb_fixedcharge.py b/tests/test_arcadia_urdb_fixedcharge.py new file mode 100644 index 0000000..ae3cf14 --- /dev/null +++ b/tests/test_arcadia_urdb_fixedcharge.py @@ -0,0 +1,27 @@ +from datetime import datetime + +from tariff_fetch.urdb.arcadia.fixedcharge import get_rate_fixed_charge_at_dt + + +def test_get_rate_fixed_charge_returns_zero_when_no_bands_apply(monkeypatch): + rate = { + "charge_type": "FIXED_PRICE", + "transaction_type": "BUY", + "charge_period": "MONTHLY", + "tariff_rate_id": 1, + "rate_name": "Customer Charge", + } + + monkeypatch.setattr( + "tariff_fetch.urdb.arcadia.fixedcharge.ru.rate_filter_bands", + lambda rate, scenario, library: [], + ) + + result = get_rate_fixed_charge_at_dt( + scenario=None, # pyright: ignore[reportArgumentType] + library=None, # pyright: ignore[reportArgumentType] + rate=rate, # pyright: ignore[reportArgumentType] + dt=datetime(2025, 1, 1, 0, 30), + ) + + assert result == 0 diff --git a/tests/test_arcadia_urdb_golden.py b/tests/test_arcadia_urdb_golden.py new file mode 100644 index 0000000..f97c05a --- /dev/null +++ b/tests/test_arcadia_urdb_golden.py @@ -0,0 +1,94 @@ +from datetime import datetime +from math import inf +from types import SimpleNamespace + +from tariff_fetch.urdb.arcadia import energyschedule as es +from tariff_fetch.urdb.arcadia import fixedcharge as fc +from tariff_fetch.urdb.arcadia import metadata as metadata_mod +from tariff_fetch.urdb.arcadia.scenario import Scenario + + +def test_golden_flat_energy_schedule(monkeypatch): + monkeypatch.setattr(es, "get_month_hour_bands", lambda scenario, library, month, hour, day_filter: [(inf, 0.15)]) + + result = es.build_energy_schedule(Scenario(1, 2025, apply_percentages=False), library=None) # type: ignore[arg-type] + + assert result == { + "energyratestructure": [[{"rate": 0.15, "unit": "kWh"}]], + "energyweekdayschedule": tuple(tuple(0 for _ in range(24)) for _ in range(12)), + "energyweekendschedule": tuple(tuple(0 for _ in range(24)) for _ in range(12)), + } + + +def test_golden_tiered_energy_schedule(monkeypatch): + monkeypatch.setattr( + es, + "get_month_hour_bands", + lambda scenario, library, month, hour, day_filter: [(100.0, 0.1), (inf, 0.2)], + ) + + result = es.build_energy_schedule(Scenario(1, 2025, apply_percentages=False), library=None) # type: ignore[arg-type] + + assert result == { + "energyratestructure": [[{"rate": 0.1, "max": 100.0, "unit": "kWh"}, {"rate": 0.2, "unit": "kWh"}]], + "energyweekdayschedule": tuple(tuple(0 for _ in range(24)) for _ in range(12)), + "energyweekendschedule": tuple(tuple(0 for _ in range(24)) for _ in range(12)), + } + + +def test_golden_fixed_charge_build(monkeypatch): + tariff = {"rates": []} + fixed_rate = { + "charge_type": "FIXED_PRICE", + "transaction_type": "BUY", + "charge_period": "MONTHLY", + "tariff_rate_id": 1, + "rate_name": "Customer Charge", + "rate_bands": [ + { + "tariff_rate_id": 1, + "rate_unit": "COST_PER_UNIT", + "rate_amount": 12.5, + "is_credit": False, + "has_consumption_limit": False, + "has_demand_limit": False, + "has_property_limit": False, + } + ], + } + library = SimpleNamespace( + tariffs=SimpleNamespace(get_tariff_at_date=lambda master_tariff_id, dt: tariff, get_rate=lambda rate_id: fixed_rate), + variables=None, + ) + + monkeypatch.setattr(fc.ru, "tariff_iter_rates_for_dt", lambda tariff, scenario, library, dt: [fixed_rate]) + monkeypatch.setattr(fc.ru, "rate_filter_bands", lambda rate, scenario, library: list(rate["rate_bands"])) + monkeypatch.setattr(fc.ru, "rate_band_get_amount_at_datetime", lambda band, library, dt: band["rate_amount"]) + + result = fc.build_fixed_charge(Scenario(1, 2025, apply_percentages=False), library) # type: ignore[arg-type] + + assert result == { + "fixedchargefirstmeter": 12.5, + "fixedchargeunits": "$/month", + } + + +def test_golden_metadata_chunk(): + library = SimpleNamespace( + tariffs=SimpleNamespace( + get_tariff_at_date=lambda master_tariff_id, dt: { + "lse_code": "CONED", + "lse_name": "Consolidated Edison", + "tariff_name": "Service Classification 1", + } + ) + ) + + result = metadata_mod.build_metadata(Scenario(2252, 2025, apply_percentages=False), library) # type: ignore[arg-type] + + assert result == { + "label": "CONED", + "utility": "Consolidated Edison", + "name": "Service Classification 1", + "country": "USA", + } diff --git a/tests/test_arcadia_urdb_library_debug.py b/tests/test_arcadia_urdb_library_debug.py new file mode 100644 index 0000000..35b125c --- /dev/null +++ b/tests/test_arcadia_urdb_library_debug.py @@ -0,0 +1,39 @@ +import json +from datetime import date, datetime +from pathlib import Path + +from tariff_fetch.urdb.arcadia.library import LibraryDebugStore + + +def test_library_debug_store_saves_tariff_lookup_and_property(tmp_path: Path): + store = LibraryDebugStore(tmp_path / "arcadia_library") + + store.save_tariff( + { + "tariff_id": 10, + "master_tariff_id": 20, + "effective_date": date(2025, 1, 1), + } # type: ignore[reportArgumentType] + ) + store.save_lookups( + "supplyCharge", + 2025, + [ + { + "from_date_time": datetime(2025, 1, 1, 0, 0), + "to_date_time": datetime(2025, 2, 1, 0, 0), + "actual_value": 0.1, + "best_value": None, + "forecast_value": None, + } + ], # type: ignore[reportArgumentType] + ) + store.save_property_value("territoryId", ["1", "2"]) + + tariff_path = tmp_path / "arcadia_library" / "tariffs" / "master-20_tariff-10_effective-2025-01-01.json" + lookups_path = tmp_path / "arcadia_library" / "lookups" / "supplyCharge_2025.json" + property_path = tmp_path / "arcadia_library" / "properties" / "territoryId.json" + + assert json.loads(tariff_path.read_text())["tariff_id"] == 10 + assert json.loads(lookups_path.read_text())[0]["actual_value"] == 0.1 + assert json.loads(property_path.read_text()) == {"key": "territoryId", "value": ["1", "2"]} diff --git a/tests/test_arcadia_urdb_rateutils.py b/tests/test_arcadia_urdb_rateutils.py new file mode 100644 index 0000000..1ff683d --- /dev/null +++ b/tests/test_arcadia_urdb_rateutils.py @@ -0,0 +1,216 @@ +from dataclasses import dataclass +from datetime import datetime +from math import inf +from types import SimpleNamespace + +import pytest + +from tariff_fetch.urdb.arcadia import energyschedule as es +from tariff_fetch.urdb.arcadia import rateutils as ru +from tariff_fetch.urdb.arcadia.exception import RateConversionError +from tariff_fetch.urdb.arcadia.scenario import Scenario + + +@dataclass +class _StubTariffLibrary: + properties: dict[str, dict] # pyright: ignore[reportMissingTypeArgument] + + def get_property(self, key: str): + return self.properties[key] + + +class _StubLibrary: + def __init__( + self, *, properties: dict[str, object] | None = None, tariff_properties: dict[str, dict] | None = None + ): + self._properties = properties or {} + self.tariffs = _StubTariffLibrary(tariff_properties or {}) + + def get_property(self, key: str, data_type: str): + return self._properties[key] + + def get_choice_property_as_ints(self, key: str) -> list[int]: + values = self._properties[key] + assert isinstance(values, list) + return [int(value) for value in values] + + +def test_rate_filter_bands_excludes_non_matching_choice_band(): + rate = { + "rate_bands": [ + {"applicability_value": "B", "rate_unit": "COST_PER_UNIT"}, + ], + "applicability_key": "serviceVoltage", + } + library = _StubLibrary( + properties={"serviceVoltage": ["A"]}, + tariff_properties={ + "serviceVoltage": { + "key_name": "serviceVoltage", + "property_types": "APPLICABILITY", + "operator": "=", + "data_type": "CHOICE", + } + }, + ) + + result = ru.rate_filter_bands(rate, Scenario(1, 2025, False), library) # type: ignore[arg-type] + + assert result == [] + + +def test_rate_filter_bands_includes_matching_boolean_band(): + band = {"applicability_value": "true", "rate_unit": "COST_PER_UNIT"} + rate = { + "rate_bands": [band], + "applicability_key": "isSolar", + } + library = _StubLibrary( + properties={"isSolar": True}, + tariff_properties={ + "isSolar": { + "key_name": "isSolar", + "property_types": "RATE_CRITERIA", + "operator": "=", + "data_type": "BOOLEAN", + } + }, + ) + + result = ru.rate_filter_bands(rate, Scenario(1, 2025, False), library) # type: ignore[arg-type] + + assert result == [band] + + +def test_rate_filter_bands_rejects_unsupported_operator(): + rate = { + "rate_bands": [{"applicability_value": "A", "rate_unit": "COST_PER_UNIT"}], + "applicability_key": "serviceVoltage", + } + library = _StubLibrary( + properties={"serviceVoltage": ["A"]}, + tariff_properties={ + "serviceVoltage": { + "key_name": "serviceVoltage", + "property_types": "APPLICABILITY", + "operator": "!=", + "data_type": "CHOICE", + } + }, + ) + + with pytest.raises(RateConversionError, match="Only `=` operators are supported"): + ru.rate_filter_bands(rate, Scenario(1, 2025, False), library) # type: ignore[arg-type] + + +def test_rate_is_applied_to_scenario_filters_territory(): + rate = { + "charge_class": ["SUPPLY"], + "territory": {"territory_id": 2}, + } + scenario = Scenario(1, 2025, False, {"SUPPLY"}) + library = _StubLibrary(properties={"territoryId": ["1"]}) + + result = ru.rate_is_applied_to_scenario(rate, scenario, library) # type: ignore[arg-type] + + assert result is False + + +def test_get_raw_bands_at_datetime_applies_matching_percentage(monkeypatch): + consumption_rate = { + "charge_type": "CONSUMPTION_BASED", + "transaction_type": "BUY", + "charge_class": ["SUPPLY"], + "charge_period": "MONTHLY", + "rate_bands": [ + { + "tariff_rate_id": 1, + "rate_unit": "COST_PER_UNIT", + "rate_amount": 10.0, + "is_credit": False, + "consumption_upper_limit": inf, + } + ], + } + percentage_rate = { + "charge_type": "FIXED_PRICE", + "transaction_type": "BUY", + "charge_class": ["SUPPLY"], + "charge_period": "MONTHLY", + "rate_bands": [ + { + "tariff_rate_id": 2, + "rate_unit": "PERCENTAGE", + "rate_amount": 10.0, + "is_credit": False, + } + ], + } + library = SimpleNamespace( + tariffs=SimpleNamespace(get_tariff_at_date=lambda master_tariff_id, dt: {"rates": []}), + variables=None, + ) + + monkeypatch.setattr( + es.ru, "tariff_iter_rates_for_dt", lambda tariff, scenario, library, dt: [consumption_rate, percentage_rate] + ) + monkeypatch.setattr(es.ru, "rate_filter_bands", lambda rate, scenario, library: list(rate["rate_bands"])) + monkeypatch.setattr(es.ru, "rate_band_get_amount_at_datetime", lambda band, library, dt: band["rate_amount"]) + + result = es.get_raw_bands_at_datetime( + Scenario(1, 2025, apply_percentages=True, charge_classes={"SUPPLY"}), + library, # pyright: ignore[reportArgumentType] + datetime(2025, 1, 1, 0, 30), + ) + + assert result == [(inf, 11.0)] + + +def test_get_raw_bands_at_datetime_skips_percentage_when_disabled(monkeypatch): + consumption_rate = { + "charge_type": "CONSUMPTION_BASED", + "transaction_type": "BUY", + "charge_class": ["SUPPLY"], + "charge_period": "MONTHLY", + "rate_bands": [ + { + "tariff_rate_id": 1, + "rate_unit": "COST_PER_UNIT", + "rate_amount": 10.0, + "is_credit": False, + "consumption_upper_limit": inf, + } + ], + } + percentage_rate = { + "charge_type": "FIXED_PRICE", + "transaction_type": "BUY", + "charge_class": ["SUPPLY"], + "charge_period": "MONTHLY", + "rate_bands": [ + { + "tariff_rate_id": 2, + "rate_unit": "PERCENTAGE", + "rate_amount": 10.0, + "is_credit": False, + } + ], + } + library = SimpleNamespace( + tariffs=SimpleNamespace(get_tariff_at_date=lambda master_tariff_id, dt: {"rates": []}), + variables=None, + ) + + monkeypatch.setattr( + es.ru, "tariff_iter_rates_for_dt", lambda tariff, scenario, library, dt: [consumption_rate, percentage_rate] + ) + monkeypatch.setattr(es.ru, "rate_filter_bands", lambda rate, scenario, library: list(rate["rate_bands"])) + monkeypatch.setattr(es.ru, "rate_band_get_amount_at_datetime", lambda band, library, dt: band["rate_amount"]) + + result = es.get_raw_bands_at_datetime( + Scenario(1, 2025, apply_percentages=False, charge_classes={"SUPPLY"}), + library, # pyright: ignore[reportArgumentType] + datetime(2025, 1, 1, 0, 30), + ) + + assert result == [(inf, 10.0)] diff --git a/tests/test_rateacuity_urdb.py b/tests/test_rateacuity_urdb.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/tests/test_rateacuity_urdb.py @@ -0,0 +1 @@ + From 8e4b0829924789ab721ea1f2c230e48a58b5d04c Mon Sep 17 00:00:00 2001 From: Natalia Borovskikh Date: Thu, 12 Mar 2026 13:04:30 +0000 Subject: [PATCH 11/24] support daily fixed charges --- tariff_fetch/urdb/arcadia/fixedcharge.py | 14 ++++++-- tests/test_arcadia_urdb_fixedcharge.py | 46 +++++++++++++++++++++++- tests/test_arcadia_urdb_golden.py | 38 ++++++++++++++++++++ 3 files changed, 95 insertions(+), 3 deletions(-) diff --git a/tariff_fetch/urdb/arcadia/fixedcharge.py b/tariff_fetch/urdb/arcadia/fixedcharge.py index 0813fef..f3141da 100644 --- a/tariff_fetch/urdb/arcadia/fixedcharge.py +++ b/tariff_fetch/urdb/arcadia/fixedcharge.py @@ -1,3 +1,4 @@ +import calendar from collections.abc import Iterator from datetime import datetime, timedelta from statistics import mean @@ -44,8 +45,8 @@ def get_rate_fixed_charge_at_dt(scenario: Scenario, library: Library, rate: Tari ) if "COST_PER_UNIT" not in band_rate_units: raise RateConversionError(rate, "Fixed price rate bands units should be COST_PER_UNIT") - if (charge_period := rate["charge_period"]) != "MONTHLY": - raise RateConversionError(rate, f"Fixed charges should be monthly (got {charge_period})") + if (charge_period := rate["charge_period"]) not in {"MONTHLY", "DAILY"}: + raise RateConversionError(rate, f"Fixed charges should be monthly or daily (got {charge_period})") if len(band_rate_units) > 1: raise RateConversionError(rate, "More than one applicable band for percentage rate") band = bands[0] @@ -67,12 +68,21 @@ def get_rate_fixed_charge_at_dt(scenario: Scenario, library: Library, rate: Tari raise RateConversionError(rate, "Fixed rate bands cannot have applicability_formula") rate_amount = ru.rate_band_get_amount_at_datetime(band, library, dt) + rate_amount = normalize_fixed_charge_amount(rate_amount, charge_period, dt) if rate["tariff_rate_id"] not in _LOGGED: print(f"Applied fixed charge: {rate['rate_name']} ({rate['tariff_rate_id']}) ({rate_amount})") _LOGGED.add(rate["tariff_rate_id"]) return rate_amount +def normalize_fixed_charge_amount(rate_amount: float, charge_period: str, dt: datetime) -> float: + if charge_period == "MONTHLY": + return rate_amount + if charge_period == "DAILY": + return rate_amount * calendar.monthrange(dt.year, dt.month)[1] + raise ValueError(f"Unsupported fixed charge period: {charge_period}") + + def _iter_year(year: int, delta: timedelta) -> Iterator[datetime]: dt = datetime(year, 1, 1, 0, 30, 0) max_dt = datetime(year + 1, 1, 1, 0, 0, 0) diff --git a/tests/test_arcadia_urdb_fixedcharge.py b/tests/test_arcadia_urdb_fixedcharge.py index ae3cf14..2f9c58c 100644 --- a/tests/test_arcadia_urdb_fixedcharge.py +++ b/tests/test_arcadia_urdb_fixedcharge.py @@ -1,6 +1,8 @@ from datetime import datetime -from tariff_fetch.urdb.arcadia.fixedcharge import get_rate_fixed_charge_at_dt +import pytest + +from tariff_fetch.urdb.arcadia.fixedcharge import get_rate_fixed_charge_at_dt, normalize_fixed_charge_amount def test_get_rate_fixed_charge_returns_zero_when_no_bands_apply(monkeypatch): @@ -25,3 +27,45 @@ def test_get_rate_fixed_charge_returns_zero_when_no_bands_apply(monkeypatch): ) assert result == 0 + + +def test_get_rate_fixed_charge_converts_daily_to_monthly(monkeypatch): + rate = { + "charge_type": "FIXED_PRICE", + "transaction_type": "BUY", + "charge_period": "DAILY", + "tariff_rate_id": 1, + "rate_name": "Customer Charge", + "rate_bands": [ + { + "rate_unit": "COST_PER_UNIT", + "rate_amount": 2.0, + "has_consumption_limit": False, + "has_demand_limit": False, + "has_property_limit": False, + } + ], + } + + monkeypatch.setattr( + "tariff_fetch.urdb.arcadia.fixedcharge.ru.rate_filter_bands", + lambda rate, scenario, library: list(rate["rate_bands"]), + ) + monkeypatch.setattr( + "tariff_fetch.urdb.arcadia.fixedcharge.ru.rate_band_get_amount_at_datetime", + lambda band, library, dt: band["rate_amount"], + ) + + result = get_rate_fixed_charge_at_dt( + scenario=None, # pyright: ignore[reportArgumentType] + library=None, # pyright: ignore[reportArgumentType] + rate=rate, # pyright: ignore[reportArgumentType] + dt=datetime(2025, 2, 1, 0, 30), + ) + + assert result == 56.0 + + +def test_normalize_fixed_charge_amount_rejects_unsupported_period(): + with pytest.raises(ValueError, match="Unsupported fixed charge period"): + normalize_fixed_charge_amount(1.0, "YEARLY", datetime(2025, 1, 1, 0, 30)) diff --git a/tests/test_arcadia_urdb_golden.py b/tests/test_arcadia_urdb_golden.py index f97c05a..e647c86 100644 --- a/tests/test_arcadia_urdb_golden.py +++ b/tests/test_arcadia_urdb_golden.py @@ -73,6 +73,44 @@ def test_golden_fixed_charge_build(monkeypatch): } +def test_golden_daily_fixed_charge_build(monkeypatch): + tariff = {"rates": []} + fixed_rate = { + "charge_type": "FIXED_PRICE", + "transaction_type": "BUY", + "charge_period": "DAILY", + "tariff_rate_id": 1, + "rate_name": "Customer Charge", + "rate_bands": [ + { + "tariff_rate_id": 1, + "rate_unit": "COST_PER_UNIT", + "rate_amount": 1.0, + "is_credit": False, + "has_consumption_limit": False, + "has_demand_limit": False, + "has_property_limit": False, + } + ], + } + library = SimpleNamespace( + tariffs=SimpleNamespace(get_tariff_at_date=lambda master_tariff_id, dt: tariff, get_rate=lambda rate_id: fixed_rate), + variables=None, + ) + + monkeypatch.setattr(fc.ru, "tariff_iter_rates_for_dt", lambda tariff, scenario, library, dt: [fixed_rate]) + monkeypatch.setattr(fc.ru, "rate_filter_bands", lambda rate, scenario, library: list(rate["rate_bands"])) + monkeypatch.setattr(fc.ru, "rate_band_get_amount_at_datetime", lambda band, library, dt: band["rate_amount"]) + + result = fc.get_fixed_charge_at_dt( + Scenario(1, 2025, apply_percentages=False), + library, # type: ignore[arg-type] + datetime(2025, 4, 1, 0, 30), + ) + + assert result == 30.0 + + def test_golden_metadata_chunk(): library = SimpleNamespace( tariffs=SimpleNamespace( From 9c7d7fc9b40859f16af25eabb82cb441bf7262f9 Mon Sep 17 00:00:00 2001 From: Natalia Borovskikh Date: Thu, 12 Mar 2026 13:16:16 +0000 Subject: [PATCH 12/24] fix wrong property type expectations --- tariff_fetch/urdb/arcadia/library.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tariff_fetch/urdb/arcadia/library.py b/tariff_fetch/urdb/arcadia/library.py index 5ef0294..ecddabd 100644 --- a/tariff_fetch/urdb/arcadia/library.py +++ b/tariff_fetch/urdb/arcadia/library.py @@ -267,12 +267,12 @@ def _get_property(self, key: str, data_type: TariffPropertyPrunedDataType) -> Pr raise ConversionError(f"Value for property {key} is expected to be `float`, not {value_type}") return value case "INTEGER": - if not isinstance(value, bool): + if not isinstance(value, int): raise ConversionError(f"Value for property {key} is expected to be `int`, not {value_type}") return value case "DEMAND": - if not isinstance(value, bool): + if not isinstance(value, float): raise ConversionError(f"Value for property {key} is expected to be `float`, not {value_type}") return value From 4358a82c005ce2dc84d66644f6d2aa876bd9d736 Mon Sep 17 00:00:00 2001 From: Natalia Borovskikh Date: Thu, 12 Mar 2026 13:25:40 +0000 Subject: [PATCH 13/24] Fix formatting issues --- pyproject.toml | 4 +- tests/test_arcadia_urdb_boundaries_build.py | 18 +++- tests/test_arcadia_urdb_fixedcharge.py | 26 +---- tests/test_arcadia_urdb_golden.py | 51 +-------- tests/test_arcadia_urdb_rateutils.py | 108 +++----------------- tests/test_rateacuity_urdb.py | 1 - uv.lock | 2 +- 7 files changed, 43 insertions(+), 167 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 62312e7..d8d9471 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "tariff_fetch" -version = "0.4.5" +version = "0.4.6" description = "A CLI tool and a python library to simplify downloading electric and gas utility tariff data." authors = [{ name = "Switchbox", email = "hello@switch.box" }] readme = "README.md" @@ -90,6 +90,8 @@ reportUnknownParameterType = "none" reportUnknownLambdaType = "none" reportUnknownArgumentType = "none" reportArgumentType = "none" +reportAny = "none" +reportExplicitAny = "none" reportMissingParameterType = "none" reportMissingTypeArgument = "none" reportMissingTypeStubs = "none" diff --git a/tests/test_arcadia_urdb_boundaries_build.py b/tests/test_arcadia_urdb_boundaries_build.py index 5daae78..2a6bca6 100644 --- a/tests/test_arcadia_urdb_boundaries_build.py +++ b/tests/test_arcadia_urdb_boundaries_build.py @@ -78,9 +78,21 @@ def test_build_urdb_merges_converter_chunks(monkeypatch): scenario = Scenario(123, 2025, apply_percentages=True, charge_classes={"SUPPLY"}) monkeypatch.setattr(build_mod, "Library", lambda api: SimpleNamespace()) - monkeypatch.setattr(build_mod, "build_energy_schedule", lambda scenario, library: {"energyratestructure": [[{"rate": 1.0, "unit": "kWh"}]]}) - monkeypatch.setattr(build_mod, "build_fixed_charge", lambda scenario, library: {"fixedchargefirstmeter": 12.0, "fixedchargeunits": "$/month"}) - monkeypatch.setattr(build_mod, "build_metadata", lambda scenario, library: {"label": "UTIL", "utility": "Utility", "name": "Tariff", "country": "USA"}) + monkeypatch.setattr( + build_mod, + "build_energy_schedule", + lambda scenario, library: {"energyratestructure": [[{"rate": 1.0, "unit": "kWh"}]]}, + ) + monkeypatch.setattr( + build_mod, + "build_fixed_charge", + lambda scenario, library: {"fixedchargefirstmeter": 12.0, "fixedchargeunits": "$/month"}, + ) + monkeypatch.setattr( + build_mod, + "build_metadata", + lambda scenario, library: {"label": "UTIL", "utility": "Utility", "name": "Tariff", "country": "USA"}, + ) result = build_mod.build_urdb(api=object(), scenario=scenario) # type: ignore[arg-type] diff --git a/tests/test_arcadia_urdb_fixedcharge.py b/tests/test_arcadia_urdb_fixedcharge.py index 2f9c58c..14b48a9 100644 --- a/tests/test_arcadia_urdb_fixedcharge.py +++ b/tests/test_arcadia_urdb_fixedcharge.py @@ -3,16 +3,11 @@ import pytest from tariff_fetch.urdb.arcadia.fixedcharge import get_rate_fixed_charge_at_dt, normalize_fixed_charge_amount +from tests.arcadia_urdb_fixtures import make_band, make_fixed_rate def test_get_rate_fixed_charge_returns_zero_when_no_bands_apply(monkeypatch): - rate = { - "charge_type": "FIXED_PRICE", - "transaction_type": "BUY", - "charge_period": "MONTHLY", - "tariff_rate_id": 1, - "rate_name": "Customer Charge", - } + rate = make_fixed_rate(rate_bands=[]) monkeypatch.setattr( "tariff_fetch.urdb.arcadia.fixedcharge.ru.rate_filter_bands", @@ -30,22 +25,7 @@ def test_get_rate_fixed_charge_returns_zero_when_no_bands_apply(monkeypatch): def test_get_rate_fixed_charge_converts_daily_to_monthly(monkeypatch): - rate = { - "charge_type": "FIXED_PRICE", - "transaction_type": "BUY", - "charge_period": "DAILY", - "tariff_rate_id": 1, - "rate_name": "Customer Charge", - "rate_bands": [ - { - "rate_unit": "COST_PER_UNIT", - "rate_amount": 2.0, - "has_consumption_limit": False, - "has_demand_limit": False, - "has_property_limit": False, - } - ], - } + rate = make_fixed_rate(charge_period="DAILY", rate_bands=[make_band(rate_amount=2.0)]) monkeypatch.setattr( "tariff_fetch.urdb.arcadia.fixedcharge.ru.rate_filter_bands", diff --git a/tests/test_arcadia_urdb_golden.py b/tests/test_arcadia_urdb_golden.py index e647c86..65a0451 100644 --- a/tests/test_arcadia_urdb_golden.py +++ b/tests/test_arcadia_urdb_golden.py @@ -6,6 +6,7 @@ from tariff_fetch.urdb.arcadia import fixedcharge as fc from tariff_fetch.urdb.arcadia import metadata as metadata_mod from tariff_fetch.urdb.arcadia.scenario import Scenario +from tests.arcadia_urdb_fixtures import make_band, make_fixed_rate, make_library_with_tariff def test_golden_flat_energy_schedule(monkeypatch): @@ -37,29 +38,8 @@ def test_golden_tiered_energy_schedule(monkeypatch): def test_golden_fixed_charge_build(monkeypatch): - tariff = {"rates": []} - fixed_rate = { - "charge_type": "FIXED_PRICE", - "transaction_type": "BUY", - "charge_period": "MONTHLY", - "tariff_rate_id": 1, - "rate_name": "Customer Charge", - "rate_bands": [ - { - "tariff_rate_id": 1, - "rate_unit": "COST_PER_UNIT", - "rate_amount": 12.5, - "is_credit": False, - "has_consumption_limit": False, - "has_demand_limit": False, - "has_property_limit": False, - } - ], - } - library = SimpleNamespace( - tariffs=SimpleNamespace(get_tariff_at_date=lambda master_tariff_id, dt: tariff, get_rate=lambda rate_id: fixed_rate), - variables=None, - ) + fixed_rate = make_fixed_rate(rate_bands=[make_band(rate_amount=12.5)]) + library = make_library_with_tariff(fixed_rate) monkeypatch.setattr(fc.ru, "tariff_iter_rates_for_dt", lambda tariff, scenario, library, dt: [fixed_rate]) monkeypatch.setattr(fc.ru, "rate_filter_bands", lambda rate, scenario, library: list(rate["rate_bands"])) @@ -74,29 +54,8 @@ def test_golden_fixed_charge_build(monkeypatch): def test_golden_daily_fixed_charge_build(monkeypatch): - tariff = {"rates": []} - fixed_rate = { - "charge_type": "FIXED_PRICE", - "transaction_type": "BUY", - "charge_period": "DAILY", - "tariff_rate_id": 1, - "rate_name": "Customer Charge", - "rate_bands": [ - { - "tariff_rate_id": 1, - "rate_unit": "COST_PER_UNIT", - "rate_amount": 1.0, - "is_credit": False, - "has_consumption_limit": False, - "has_demand_limit": False, - "has_property_limit": False, - } - ], - } - library = SimpleNamespace( - tariffs=SimpleNamespace(get_tariff_at_date=lambda master_tariff_id, dt: tariff, get_rate=lambda rate_id: fixed_rate), - variables=None, - ) + fixed_rate = make_fixed_rate(charge_period="DAILY", rate_bands=[make_band(rate_amount=1.0)]) + library = make_library_with_tariff(fixed_rate) monkeypatch.setattr(fc.ru, "tariff_iter_rates_for_dt", lambda tariff, scenario, library, dt: [fixed_rate]) monkeypatch.setattr(fc.ru, "rate_filter_bands", lambda rate, scenario, library: list(rate["rate_bands"])) diff --git a/tests/test_arcadia_urdb_rateutils.py b/tests/test_arcadia_urdb_rateutils.py index 1ff683d..d9dacb6 100644 --- a/tests/test_arcadia_urdb_rateutils.py +++ b/tests/test_arcadia_urdb_rateutils.py @@ -1,4 +1,3 @@ -from dataclasses import dataclass from datetime import datetime from math import inf from types import SimpleNamespace @@ -9,40 +8,15 @@ from tariff_fetch.urdb.arcadia import rateutils as ru from tariff_fetch.urdb.arcadia.exception import RateConversionError from tariff_fetch.urdb.arcadia.scenario import Scenario - - -@dataclass -class _StubTariffLibrary: - properties: dict[str, dict] # pyright: ignore[reportMissingTypeArgument] - - def get_property(self, key: str): - return self.properties[key] - - -class _StubLibrary: - def __init__( - self, *, properties: dict[str, object] | None = None, tariff_properties: dict[str, dict] | None = None - ): - self._properties = properties or {} - self.tariffs = _StubTariffLibrary(tariff_properties or {}) - - def get_property(self, key: str, data_type: str): - return self._properties[key] - - def get_choice_property_as_ints(self, key: str) -> list[int]: - values = self._properties[key] - assert isinstance(values, list) - return [int(value) for value in values] +from tests.arcadia_urdb_fixtures import StubLibrary, make_band, make_consumption_rate, make_percentage_rate def test_rate_filter_bands_excludes_non_matching_choice_band(): rate = { - "rate_bands": [ - {"applicability_value": "B", "rate_unit": "COST_PER_UNIT"}, - ], + "rate_bands": [make_band(applicability_value="B")], "applicability_key": "serviceVoltage", } - library = _StubLibrary( + library = StubLibrary( properties={"serviceVoltage": ["A"]}, tariff_properties={ "serviceVoltage": { @@ -60,12 +34,12 @@ def test_rate_filter_bands_excludes_non_matching_choice_band(): def test_rate_filter_bands_includes_matching_boolean_band(): - band = {"applicability_value": "true", "rate_unit": "COST_PER_UNIT"} + band = make_band(applicability_value="true") rate = { "rate_bands": [band], "applicability_key": "isSolar", } - library = _StubLibrary( + library = StubLibrary( properties={"isSolar": True}, tariff_properties={ "isSolar": { @@ -84,10 +58,10 @@ def test_rate_filter_bands_includes_matching_boolean_band(): def test_rate_filter_bands_rejects_unsupported_operator(): rate = { - "rate_bands": [{"applicability_value": "A", "rate_unit": "COST_PER_UNIT"}], + "rate_bands": [make_band(applicability_value="A")], "applicability_key": "serviceVoltage", } - library = _StubLibrary( + library = StubLibrary( properties={"serviceVoltage": ["A"]}, tariff_properties={ "serviceVoltage": { @@ -109,7 +83,7 @@ def test_rate_is_applied_to_scenario_filters_territory(): "territory": {"territory_id": 2}, } scenario = Scenario(1, 2025, False, {"SUPPLY"}) - library = _StubLibrary(properties={"territoryId": ["1"]}) + library = StubLibrary(properties={"territoryId": ["1"]}) result = ru.rate_is_applied_to_scenario(rate, scenario, library) # type: ignore[arg-type] @@ -117,35 +91,10 @@ def test_rate_is_applied_to_scenario_filters_territory(): def test_get_raw_bands_at_datetime_applies_matching_percentage(monkeypatch): - consumption_rate = { - "charge_type": "CONSUMPTION_BASED", - "transaction_type": "BUY", - "charge_class": ["SUPPLY"], - "charge_period": "MONTHLY", - "rate_bands": [ - { - "tariff_rate_id": 1, - "rate_unit": "COST_PER_UNIT", - "rate_amount": 10.0, - "is_credit": False, - "consumption_upper_limit": inf, - } - ], - } - percentage_rate = { - "charge_type": "FIXED_PRICE", - "transaction_type": "BUY", - "charge_class": ["SUPPLY"], - "charge_period": "MONTHLY", - "rate_bands": [ - { - "tariff_rate_id": 2, - "rate_unit": "PERCENTAGE", - "rate_amount": 10.0, - "is_credit": False, - } - ], - } + consumption_rate = make_consumption_rate(rate_bands=[make_band(rate_amount=10.0, consumption_upper_limit=inf)]) + percentage_rate = make_percentage_rate( + rate_bands=[make_band(tariff_rate_id=2, rate_unit="PERCENTAGE", rate_amount=10.0)] + ) library = SimpleNamespace( tariffs=SimpleNamespace(get_tariff_at_date=lambda master_tariff_id, dt: {"rates": []}), variables=None, @@ -167,35 +116,10 @@ def test_get_raw_bands_at_datetime_applies_matching_percentage(monkeypatch): def test_get_raw_bands_at_datetime_skips_percentage_when_disabled(monkeypatch): - consumption_rate = { - "charge_type": "CONSUMPTION_BASED", - "transaction_type": "BUY", - "charge_class": ["SUPPLY"], - "charge_period": "MONTHLY", - "rate_bands": [ - { - "tariff_rate_id": 1, - "rate_unit": "COST_PER_UNIT", - "rate_amount": 10.0, - "is_credit": False, - "consumption_upper_limit": inf, - } - ], - } - percentage_rate = { - "charge_type": "FIXED_PRICE", - "transaction_type": "BUY", - "charge_class": ["SUPPLY"], - "charge_period": "MONTHLY", - "rate_bands": [ - { - "tariff_rate_id": 2, - "rate_unit": "PERCENTAGE", - "rate_amount": 10.0, - "is_credit": False, - } - ], - } + consumption_rate = make_consumption_rate(rate_bands=[make_band(rate_amount=10.0, consumption_upper_limit=inf)]) + percentage_rate = make_percentage_rate( + rate_bands=[make_band(tariff_rate_id=2, rate_unit="PERCENTAGE", rate_amount=10.0)] + ) library = SimpleNamespace( tariffs=SimpleNamespace(get_tariff_at_date=lambda master_tariff_id, dt: {"rates": []}), variables=None, diff --git a/tests/test_rateacuity_urdb.py b/tests/test_rateacuity_urdb.py index 8b13789..e69de29 100644 --- a/tests/test_rateacuity_urdb.py +++ b/tests/test_rateacuity_urdb.py @@ -1 +0,0 @@ - diff --git a/uv.lock b/uv.lock index 622a719..9766cdb 100644 --- a/uv.lock +++ b/uv.lock @@ -1480,7 +1480,7 @@ wheels = [ [[package]] name = "tariff-fetch" -version = "0.4.5" +version = "0.4.6" source = { editable = "." } dependencies = [ { name = "fastexcel" }, From 8f22929aee26e87eed4dd9c86ff069b10355affb Mon Sep 17 00:00:00 2001 From: Natalia Borovskikh Date: Thu, 12 Mar 2026 13:56:41 +0000 Subject: [PATCH 14/24] add documentation --- docs/providers/arcadia/index.md | 1 + docs/providers/arcadia/urdb-converter.md | 124 ++++++++++++++++++++ mkdocs.yml | 1 + tariff_fetch/urdb/arcadia/build.py | 11 +- tariff_fetch/urdb/arcadia/energyschedule.py | 24 +++- tariff_fetch/urdb/arcadia/exception.py | 14 ++- tariff_fetch/urdb/arcadia/fixedcharge.py | 16 +++ tariff_fetch/urdb/arcadia/library.py | 54 +++++++++ tariff_fetch/urdb/arcadia/metadata.py | 4 + tariff_fetch/urdb/arcadia/prompts.py | 24 ++++ tariff_fetch/urdb/arcadia/rateutils.py | 34 +++++- tariff_fetch/urdb/arcadia/scenario.py | 4 + tariff_fetch/urdb/arcadia/shared.py | 12 ++ tariff_fetch/urdb/arcadia/types.py | 2 + tests/test_arcadia_urdb_boundaries_build.py | 28 +++++ tests/test_arcadia_urdb_fixedcharge.py | 13 ++ tests/test_arcadia_urdb_rateutils.py | 35 ++++++ 17 files changed, 391 insertions(+), 10 deletions(-) create mode 100644 docs/providers/arcadia/urdb-converter.md diff --git a/docs/providers/arcadia/index.md b/docs/providers/arcadia/index.md index 601c8c4..d0dc5ef 100644 --- a/docs/providers/arcadia/index.md +++ b/docs/providers/arcadia/index.md @@ -15,6 +15,7 @@ Arcadia's Signal Tariffs API (formerly Genability) provides comprehensive US uti The Signal API uses a custom JSON objects to represent tariffs, with a complex hierarchical structure. +- **[Arcadia to URDB converter](urdb-converter.md)** - **[Short guide to the Tariff JSON](tariff-json-structure.md)** - [Official API docs on Tariff JSON](https://docs.arcadia.com/v2022-12-21-Signal/reference/tariff) diff --git a/docs/providers/arcadia/urdb-converter.md b/docs/providers/arcadia/urdb-converter.md new file mode 100644 index 0000000..acb0198 --- /dev/null +++ b/docs/providers/arcadia/urdb-converter.md @@ -0,0 +1,124 @@ +# Arcadia to URDB Converter + +This page explains what the Arcadia-to-URDB converter does, what it asks for, and which Arcadia tariffs it can convert reliably today. + +The goal of the converter is to turn Arcadia electricity tariffs into a URDB-style rate record that is useful for downstream analysis. It is intentionally conservative: if the tariff uses features that are not yet handled safely, the converter should stop instead of silently producing a misleading result. + +## What it converts + +The converter is designed for Arcadia electricity tariffs with: + +- energy charges based on kWh +- optional time-of-use structure +- optional tiered consumption bands +- a customer fixed charge +- optional percentage-based adjustments that can be applied on top of matching charges + +The result is a URDB-style record with: + +- tariff name +- utility name +- weekday schedule +- weekend schedule +- energy rate structure +- fixed monthly charge + +## What the converter asks for + +When converting a tariff, you may be asked for: + +- the tariff to convert +- the year to convert it for +- which charge classes to include +- whether percentage-based rates should be applied +- any Arcadia tariff property values needed to resolve the tariff + +Some Arcadia tariffs depend on user inputs such as territory or service configuration. If those values are required and not already known, the converter prompts for them interactively. + +## How the output should be understood + +The converter produces a URDB-style approximation of the Arcadia tariff for a chosen year. + +That matters because Arcadia tariffs can contain more detail than the URDB schedule model can represent directly. To bridge that gap, the converter samples the tariff across the year and collapses the result into: + +- one weekday schedule for each month +- one weekend schedule for each month +- one fixed monthly charge + +For many residential and other relatively simple tariffs, this is a reasonable representation. For more complex tariffs, not every Arcadia detail can be preserved exactly. + +## Supported tariff behavior + +The converter currently works best for tariffs with: + +- standard kWh-based energy charges +- monthly billing periods +- seasonal rates +- weekday/weekend time-of-use patterns +- tiered energy rates based on consumption thresholds +- monthly customer charges +- daily customer charges that can be converted to a monthly equivalent +- simple percentage adders applied to supported charge classes + +## Current limitations + +Some Arcadia tariff features are not supported yet. The converter is expected to stop when they appear instead of guessing. + +Examples include: + +- rates whose tier limits come from a variable lookup +- rates whose value must be multiplied by an additional variable factor +- rates that depend on a quantity such as number of meters +- calendar-driven time-of-use overrides such as holiday or event calendars +- demand-based rate bands +- property-limited bands and formula-driven band logic + +In practice, this means the converter currently targets simpler electricity tariffs first and does not claim full Arcadia schema coverage. + +## Fixed charges + +The converter outputs one fixed charge in monthly units. + +- monthly fixed charges are used directly +- daily fixed charges are converted to a monthly equivalent before being averaged into the final output + +If a tariff includes fixed-charge structures that do not fit this model, the converter should stop rather than emit an ambiguous result. + +## Percentage-based charges + +Some Arcadia tariffs include percentage-based adjustments. These can be included or excluded during conversion. + +- if percentage application is enabled, supported percentage adjustments are applied to matching charge classes +- if percentage application is disabled, those adjustments are left out + +This makes it easier to compare a cleaner base energy schedule against a schedule that includes percentage adders. + +## Debug output + +For troubleshooting, the converter saves fetched Arcadia data under `outputs/arcadia_library/` by default. + +This includes: + +- tariff data +- variable lookup data +- prompted property values + +This folder is useful when checking why a tariff converted a certain way or why the converter stopped on a particular feature. + +## Partial conversions + +The current interactive flow may allow you to continue after one part of the conversion fails. + +That is useful while developing or investigating difficult tariffs, but it also means you can end up with a partial URDB record if you explicitly choose to continue. For normal usage, treat a clean conversion with no skipped sections as the reliable result. + +## When to trust the output + +You should have the most confidence in the output when the tariff: + +- is an electricity tariff +- uses standard consumption-based charges +- has ordinary seasonal or time-of-use structure +- has a simple fixed customer charge +- does not rely on advanced Arcadia-only features + +If the tariff uses more advanced logic, the safer expectation is that the converter may stop and require additional support to be implemented first. diff --git a/mkdocs.yml b/mkdocs.yml index f5ea1c2..d70dc56 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -12,6 +12,7 @@ nav: - Arcadia (Genability): - Overview: providers/arcadia/index.md - Access & Credentials: providers/arcadia/access.md + - Arcadia to URDB Converter: providers/arcadia/urdb-converter.md - Tariff JSON Structure: providers/arcadia/tariff-json-structure.md - NREL / OpenEI: - Overview: providers/nrel/index.md diff --git a/tariff_fetch/urdb/arcadia/build.py b/tariff_fetch/urdb/arcadia/build.py index 4ccbb97..c6f895f 100644 --- a/tariff_fetch/urdb/arcadia/build.py +++ b/tariff_fetch/urdb/arcadia/build.py @@ -1,3 +1,5 @@ +"""Build URDB-style output from Arcadia tariff data for a conversion scenario.""" + from dataclasses import dataclass from typing import cast @@ -13,12 +15,9 @@ from .scenario import Scenario -@dataclass(frozen=True) -class URDBChunk: - fixed_charge: float - - def build_urdb(api: ArcadiaSignalAPI, scenario: Scenario) -> URDBRate: + """Build a URDB record by combining energy, fixed-charge, and metadata chunks.""" + library = Library(api) try: energy_schedule = build_energy_schedule(scenario, library) @@ -39,6 +38,8 @@ def build_urdb(api: ArcadiaSignalAPI, scenario: Scenario) -> URDBRate: def _confirm_proceed(e: Exception, processing: str) -> URDBRate: + """Ask whether to continue after a chunk-level conversion failure.""" + response = cast( bool | None, questionary.confirm(f"Error while converting: {processing}: {e}. Continue or print traceback and exit?").ask(), diff --git a/tariff_fetch/urdb/arcadia/energyschedule.py b/tariff_fetch/urdb/arcadia/energyschedule.py index 6697b6a..bfa08cc 100644 --- a/tariff_fetch/urdb/arcadia/energyschedule.py +++ b/tariff_fetch/urdb/arcadia/energyschedule.py @@ -1,3 +1,5 @@ +"""Convert Arcadia energy rates into URDB schedule and tier structures.""" + import itertools from collections.abc import Collection from datetime import datetime @@ -23,6 +25,8 @@ def build_energy_schedule(scenario: Scenario, library: Library) -> URDBRate: + """Build the URDB energy schedule and rate structure for one scenario year.""" + weekday_schedule_raw = [ [get_month_hour_bands(scenario, library, month, hour, is_weekday) for hour in range(24)] for month in range(1, 13) @@ -63,7 +67,8 @@ def get_month_hour_bands( hour: int, day_filter: DayPredicate, ) -> ConsumptionBandSet: - """Get raw consumption bands for specific month/hour""" + """Average the sampled energy bands for one month, hour, and day type.""" + bands = [ get_raw_bands_at_datetime(scenario, library, dt) for dt in iter_sampled_datetimes(scenario.year, month, hour, day_filter) @@ -72,7 +77,8 @@ def get_month_hour_bands( def get_raw_bands_at_datetime(scenario: Scenario, library: Library, dt: datetime) -> ConsumptionBandSet: - """Get raw tariff consumption-based bands at datetime dt""" + """Return the combined consumption bands that apply at one instant.""" + tariff = library.tariffs.get_tariff_at_date(scenario.master_tariff_id, dt) rates = list(ru.tariff_iter_rates_for_dt(tariff, scenario, library, dt)) percentage_modifiers = get_percentage_rates_at_datetime(rates, scenario, library, dt) @@ -97,8 +103,12 @@ def get_rate_consumption_bands_at_datetime( library: Library, dt: datetime, ) -> ConsumptionBandSet | None: + """Convert one applicable consumption rate into URDB-style piecewise bands.""" + if rate.get("charge_type") != "CONSUMPTION_BASED": return None + if rate.get("quantity_key") is not None: + raise RateConversionError(rate, "Rates with quantity_key are not supported for consumption conversion") if (transaction_type := rate["transaction_type"]) not in {"BUY", "NET", "BUY_IMPORT"}: raise RateConversionError( rate, @@ -122,10 +132,14 @@ def get_rate_consumption_bands_at_datetime( def get_percentage_rates_at_datetime( rates: Collection[TariffRateExtended], scenario: Scenario, library: Library, dt: datetime ) -> PercentageModifiers: + """Collect supported percentage-based modifiers that apply at one instant.""" + rates = [rate for rate in rates if ru.rate_get_band_units(rate) == {"PERCENTAGE"}] result: PercentageModifiers = [] for rate in rates: + if rate.get("quantity_key") is not None: + raise RateConversionError(rate, "Rates with quantity_key are not supported for percentage conversion") if (transaction_type := rate["transaction_type"]) not in {"BUY", "NET", "BUY_IMPORT"}: raise RateConversionError( rate, @@ -152,6 +166,8 @@ def get_percentage_rates_at_datetime( def sum_piecewise_bands(inputs: Collection[ConsumptionBandSet]) -> ConsumptionBandSet: + """Sum multiple piecewise band sets over their combined breakpoints.""" + if not inputs: return [] @@ -170,6 +186,8 @@ def value_at(bands: ConsumptionBandSet, limit: float) -> float: def average_aligned_bands(inputs: list[ConsumptionBandSet]) -> ConsumptionBandSet: + """Average multiple band sets after aligning them to common tier limits.""" + if not inputs: return [] @@ -196,6 +214,8 @@ def value_at(bands: ConsumptionBandSet, limit: float) -> float: def _energy_band_to_tier(band: ConsumptionBand) -> EnergyTier: + """Convert one internal band tuple into a URDB energy tier entry.""" + if band[0] == inf: return {"rate": band[1], "unit": "kWh"} return {"rate": band[1], "max": band[0], "unit": "kWh"} diff --git a/tariff_fetch/urdb/arcadia/exception.py b/tariff_fetch/urdb/arcadia/exception.py index 72095b0..0ac0443 100644 --- a/tariff_fetch/urdb/arcadia/exception.py +++ b/tariff_fetch/urdb/arcadia/exception.py @@ -1,3 +1,5 @@ +"""Converter-specific exception types for Arcadia to URDB translation.""" + from datetime import date from typing import final @@ -7,11 +9,13 @@ class ConversionError(Exception): - pass + """Base exception for Arcadia-to-URDB conversion failures.""" @final class TariffConversionError(ConversionError): + """Conversion error scoped to a master Arcadia tariff.""" + def __init__(self, master_tariff_id: int, msg: str) -> None: self.master_tariff_id = master_tariff_id self.msg = msg @@ -20,6 +24,8 @@ def __init__(self, master_tariff_id: int, msg: str) -> None: @final class RateConversionError(ConversionError): + """Conversion error scoped to a specific Arcadia tariff rate.""" + def __init__(self, rate: TariffRateExtended, msg: str) -> None: self.rate = rate self.msg = msg @@ -27,11 +33,13 @@ def __init__(self, rate: TariffRateExtended, msg: str) -> None: class TariffNotFoundError(ConversionError): - pass + """Base exception for tariff lookup misses.""" @final class TariffNotFoundById(ConversionError): + """Raised when a tariff version cannot be found by Arcadia tariff id.""" + def __init__(self, tariff_id: int) -> None: self.tariff_id = tariff_id super().__init__() @@ -43,6 +51,8 @@ def __str__(self) -> str: @final class TariffNotFoundByDate(ConversionError): + """Raised when no tariff version is effective for a requested date.""" + def __init__(self, master_tariff_id: int, dt: date) -> None: self.master_tariff_id = master_tariff_id self.dt = dt diff --git a/tariff_fetch/urdb/arcadia/fixedcharge.py b/tariff_fetch/urdb/arcadia/fixedcharge.py index f3141da..8e6d101 100644 --- a/tariff_fetch/urdb/arcadia/fixedcharge.py +++ b/tariff_fetch/urdb/arcadia/fixedcharge.py @@ -1,3 +1,5 @@ +"""Convert Arcadia fixed-price charges into a single URDB fixed charge.""" + import calendar from collections.abc import Iterator from datetime import datetime, timedelta @@ -15,6 +17,8 @@ def build_fixed_charge(scenario: Scenario, library: Library) -> URDBRate: + """Build the URDB fixed-charge fields for a scenario.""" + return { "fixedchargefirstmeter": get_fixed_charge_value(scenario, library), "fixedchargeunits": "$/month", @@ -22,10 +26,14 @@ def build_fixed_charge(scenario: Scenario, library: Library) -> URDBRate: def get_fixed_charge_value(scenario: Scenario, library: Library) -> float: + """Average the sampled fixed charge across the target year.""" + return mean(get_fixed_charge_at_dt(scenario, library, dt) for dt in _iter_year(scenario.year, timedelta(hours=12))) def get_fixed_charge_at_dt(scenario: Scenario, library: Library, dt: datetime) -> float: + """Sum all applicable fixed-price rates at one sampled instant.""" + master_tariff_id = scenario.master_tariff_id tariff = library.tariffs.get_tariff_at_date(master_tariff_id, dt.date()) rates = ru.tariff_iter_rates_for_dt(tariff, scenario, library, dt) @@ -33,9 +41,13 @@ def get_fixed_charge_at_dt(scenario: Scenario, library: Library, dt: datetime) - def get_rate_fixed_charge_at_dt(scenario: Scenario, library: Library, rate: TariffRateExtended, dt: datetime) -> float: + """Convert one applicable Arcadia rate into a fixed-charge amount at one instant.""" + bands = ru.rate_filter_bands(rate, scenario, library) if rate["charge_type"] != "FIXED_PRICE": return 0 + if rate.get("quantity_key") is not None: + raise RateConversionError(rate, "Rates with quantity_key are not supported for fixed charge conversion") if not bands: return 0 band_rate_units = {band["rate_unit"] for band in bands} @@ -76,6 +88,8 @@ def get_rate_fixed_charge_at_dt(scenario: Scenario, library: Library, rate: Tari def normalize_fixed_charge_amount(rate_amount: float, charge_period: str, dt: datetime) -> float: + """Normalize a supported fixed charge into monthly units.""" + if charge_period == "MONTHLY": return rate_amount if charge_period == "DAILY": @@ -84,6 +98,8 @@ def normalize_fixed_charge_amount(rate_amount: float, charge_period: str, dt: da def _iter_year(year: int, delta: timedelta) -> Iterator[datetime]: + """Yield evenly spaced sample datetimes across a calendar year.""" + dt = datetime(year, 1, 1, 0, 30, 0) max_dt = datetime(year + 1, 1, 1, 0, 0, 0) while dt < max_dt: diff --git a/tariff_fetch/urdb/arcadia/library.py b/tariff_fetch/urdb/arcadia/library.py index ecddabd..21c9e28 100644 --- a/tariff_fetch/urdb/arcadia/library.py +++ b/tariff_fetch/urdb/arcadia/library.py @@ -1,3 +1,5 @@ +"""Arcadia data access, caching, prompting, and debug persistence for conversion.""" + import json from datetime import date, datetime from pathlib import Path @@ -26,6 +28,8 @@ def _json_default(value: object) -> str: + """Serialize date-like values used in debug JSON dumps.""" + if isinstance(value, datetime | date): return value.isoformat() raise TypeError(f"Object of type {type(value).__name__} is not JSON serializable") @@ -33,6 +37,8 @@ def _json_default(value: object) -> str: @final class LibraryDebugStore: + """Persist fetched Arcadia artifacts to disk for debugging.""" + def __init__(self, root: Path = _DEFAULT_DEBUG_ROOT) -> None: self.root = root self.tariffs_dir = root / "tariffs" @@ -40,6 +46,8 @@ def __init__(self, root: Path = _DEFAULT_DEBUG_ROOT) -> None: self.properties_dir = root / "properties" def save_tariff(self, tariff: TariffExtended) -> None: + """Write one fetched tariff version to the debug output folder.""" + effective_date = tariff["effective_date"].isoformat() tariff_id = tariff["tariff_id"] master_tariff_id = tariff["master_tariff_id"] @@ -47,26 +55,36 @@ def save_tariff(self, tariff: TariffExtended) -> None: self._write_json(path, tariff) def save_lookups(self, key: str, year: int, lookups: list[Lookup]) -> None: + """Write one variable lookup timeseries to the debug output folder.""" + path = self.lookups_dir / f"{key}_{year}.json" self._write_json(path, lookups) def save_property_value(self, key: str, value: PropertyValue) -> None: + """Write one prompted property value to the debug output folder.""" + path = self.properties_dir / f"{key}.json" self._write_json(path, {"key": key, "value": value}) def _write_json(self, path: Path, payload: object) -> None: + """Create the target directory and write formatted JSON payload.""" + path.parent.mkdir(parents=True, exist_ok=True) _ = path.write_text(json.dumps(payload, indent=2, default=_json_default)) @final class TariffLibrary: + """Fetch and cache Arcadia tariff versions and related tariff metadata.""" + def __init__(self, api: ArcadiaSignalAPI, debug_store: LibraryDebugStore) -> None: self.api = api self.debug_store = debug_store self.tariffs: list[TariffExtended] = [] def get_tariff_at_date(self, master_tariff_id: int, dt: date) -> TariffExtended: + """Return the tariff version effective for a master tariff on a given date.""" + dt = dt.date() if isinstance(dt, datetime) else dt if (found := self._find_tariff_at_date(master_tariff_id, dt)) is not None: return found @@ -75,6 +93,8 @@ def get_tariff_at_date(self, master_tariff_id: int, dt: date) -> TariffExtended: return tariff def get_tariff(self, tariff_id: int) -> TariffExtended: + """Return a tariff by version-specific Arcadia tariff id.""" + if (found := self._find_tariff(tariff_id)) is not None: return found tariff = self._fetch_tariff(tariff_id) @@ -82,17 +102,25 @@ def get_tariff(self, tariff_id: int) -> TariffExtended: return tariff def get_rate(self, rate_id: int) -> TariffRateExtended: + """Return a cached rate by Arcadia tariff rate id.""" + return next( rate for tariff in self.tariffs for rate in tariff.get("rates", []) if rate["tariff_rate_id"] == rate_id ) def get_property(self, key: str) -> TariffPropertyStandard: + """Return a cached Arcadia tariff property definition by key name.""" + return next(prop for tariff in self.tariffs for prop in tariff.get("properties", []) if prop["key_name"] == key) def _find_tariff(self, tariff_id: int) -> TariffExtended | None: + """Search the in-memory tariff cache by tariff id.""" + return next((t for t in self.tariffs if t["tariff_id"] == tariff_id), None) def _find_tariff_at_date(self, master_tariff_id: int, dt: date) -> TariffExtended | None: + """Search the in-memory cache for the tariff version effective on a given date.""" + for tariff in self.tariffs: if tariff["master_tariff_id"] != master_tariff_id: continue @@ -101,10 +129,14 @@ def _find_tariff_at_date(self, master_tariff_id: int, dt: date) -> TariffExtende return None def _remember(self, tariff: TariffExtended) -> None: + """Add a fetched tariff to the cache and write its debug dump.""" + self.tariffs.append(tariff) self.debug_store.save_tariff(tariff) def _fetch_tariff(self, tariff_id: int) -> TariffExtended: + """Fetch one tariff version from Arcadia by tariff id.""" + tariffs = list( self.api.tariffs.iter_pages( fields="ext", @@ -123,6 +155,8 @@ def _fetch_tariff(self, tariff_id: int) -> TariffExtended: return tariffs[0] def _fetch_tariff_at_date(self, master_tariff_id: int, dt: date) -> TariffExtended: + """Fetch the tariff version effective for a master tariff on a given date.""" + tariffs = self.api.tariffs.iter_pages( fields="ext", master_tariff_id=master_tariff_id, @@ -138,12 +172,16 @@ def _fetch_tariff_at_date(self, master_tariff_id: int, dt: date) -> TariffExtend @final class VariablePropertyLibrary: + """Fetch and cache Arcadia variable lookup timeseries by property key and year.""" + def __init__(self, api: ArcadiaSignalAPI, debug_store: LibraryDebugStore): self.api = api self.debug_store = debug_store self.property_timeseries: dict[tuple[str, int], list[Lookup]] = {} def lookup(self, key: str, dt: datetime) -> float: + """Return the best available lookup value for a property at a datetime.""" + if (lookups := self.property_timeseries.get((key, dt.year))) is None: lookups = self._lookup_property_timeseries(key, dt.year) self.property_timeseries[(key, dt.year)] = lookups @@ -161,6 +199,8 @@ def lookup(self, key: str, dt: datetime) -> float: return 0.0 def _lookup_property_timeseries(self, key: str, year: int) -> list[Lookup]: + """Fetch one year of lookup rows for a variable Arcadia property.""" + return list( self.api.properties.lookups.iter_pages( key, @@ -172,6 +212,8 @@ def _lookup_property_timeseries(self, key: str, year: int) -> list[Lookup]: @final class Library: + """Top-level Arcadia conversion helper for tariffs, lookups, and prompted properties.""" + def __init__( self, api: ArcadiaSignalAPI, @@ -185,9 +227,13 @@ def __init__( self._properies: dict[str, PropertyValue] = properties or {} def has_property(self, key: str) -> bool: + """Return whether a property value has already been supplied or cached.""" + return key in self._properies def get_choice_property_as_ints(self, key: str) -> list[int]: + """Return a CHOICE property as integer ids.""" + strs = self.get_property(key, "CHOICE") try: return list(map(int, strs)) @@ -212,6 +258,8 @@ def get_property(self, key: str, data_type: Literal["INTEGER"]) -> int: ... def get_property(self, key: str, data_type: Literal["DEMAND"]) -> float: ... def get_property(self, key: str, data_type: TariffPropertyPrunedDataType) -> PropertyValue: + """Return a property value, prompting and caching it if needed.""" + if (found := self._get_property(key, data_type)) is not None: return found tariff_property = self.tariffs.get_property(key) @@ -240,6 +288,8 @@ def _get_property(self, key: str, data_type: Literal["INTEGER"]) -> int | None: def _get_property(self, key: str, data_type: Literal["DEMAND"]) -> float | None: ... def _get_property(self, key: str, data_type: TariffPropertyPrunedDataType) -> PropertyValue | None: + """Return a cached property value after validating its expected data type.""" + try: value = self._properies[key] except KeyError: @@ -278,12 +328,16 @@ def _get_property(self, key: str, data_type: TariffPropertyPrunedDataType) -> Pr def _is_tariff_effective_on(tariff: TariffExtended, dt: date) -> bool: + """Return whether a tariff version is effective on the given date.""" + effective_date = tariff["effective_date"] end_date = tariff["end_date"] or date.max return effective_date <= dt < end_date def _prompt_property(tariff_property: TariffPropertyStandard) -> PropertyValue | None: + """Prompt the user for a property value using the Arcadia property data type.""" + data_type = tariff_property["data_type"] match data_type: case "BOOLEAN": diff --git a/tariff_fetch/urdb/arcadia/metadata.py b/tariff_fetch/urdb/arcadia/metadata.py index 7ca4d42..7d699e4 100644 --- a/tariff_fetch/urdb/arcadia/metadata.py +++ b/tariff_fetch/urdb/arcadia/metadata.py @@ -1,3 +1,5 @@ +"""Build basic URDB metadata fields from the selected Arcadia tariff version.""" + from datetime import date from tariff_fetch.urdb.schema import URDBRate @@ -7,6 +9,8 @@ def build_metadata(scenario: Scenario, library: Library) -> URDBRate: + """Map the scenario tariff's utility and tariff names into URDB metadata fields.""" + master_tariff_id = scenario.master_tariff_id tariff = library.tariffs.get_tariff_at_date(master_tariff_id, date(scenario.year, 1, 1)) return {"label": tariff["lse_code"], "utility": tariff["lse_name"], "name": tariff["tariff_name"], "country": "USA"} diff --git a/tariff_fetch/urdb/arcadia/prompts.py b/tariff_fetch/urdb/arcadia/prompts.py index c25ad71..17ad505 100644 --- a/tariff_fetch/urdb/arcadia/prompts.py +++ b/tariff_fetch/urdb/arcadia/prompts.py @@ -1,3 +1,5 @@ +"""Interactive prompts for Arcadia conversion scenario and property inputs.""" + from datetime import date from typing import cast, get_args @@ -33,6 +35,8 @@ def prompt_charge_classes() -> set[RateChargeClass] | None: + """Prompt for the Arcadia charge classes to include in conversion.""" + choices = cast(tuple[RateChargeClass, ...], get_args(RateChargeClass)) result_raw = cast( list[RateChargeClass] | None, @@ -54,6 +58,8 @@ def prompt_charge_classes() -> set[RateChargeClass] | None: def prompt_string(tariff_property: TariffPropertyStandard) -> str | None: + """Prompt for a string-valued Arcadia tariff property.""" + default_value = tariff_property.get("property_value") if tariff_property["is_default"] else None return cast( str | None, @@ -65,6 +71,8 @@ def prompt_string(tariff_property: TariffPropertyStandard) -> str | None: def prompt_choice(tariff_property: TariffPropertyStandard) -> list[str] | None: + """Prompt for a multi-select Arcadia CHOICE property.""" + if tariff_property["is_default"]: default_value_raw = tariff_property.get("property_value") default_value = {item.strip() for item in (default_value_raw or "").split(",")} @@ -87,6 +95,8 @@ def prompt_choice(tariff_property: TariffPropertyStandard) -> list[str] | None: def prompt_boolean(tariff_property: TariffPropertyStandard) -> bool | None: + """Prompt for a boolean Arcadia tariff property.""" + default_value = tariff_property.get("property_value") if tariff_property["is_default"] else None default_value = True if default_value == "true" else (False if default_value == "false" else None) result = cast( @@ -99,10 +109,14 @@ def prompt_boolean(tariff_property: TariffPropertyStandard) -> bool | None: def prompt_date(tariff_property: TariffPropertyStandard) -> date | None: # pyright: ignore[reportUnusedParameter] + """Placeholder for date property prompting, which is not implemented yet.""" + raise NotImplementedError() def prompt_decimal(tariff_property: TariffPropertyStandard) -> float | None: + """Prompt for a decimal-valued Arcadia tariff property.""" + default_value = tariff_property.get("property_value") if tariff_property["is_default"] else None default_value = float(default_value) if default_value else None @@ -120,6 +134,8 @@ def prompt_decimal(tariff_property: TariffPropertyStandard) -> float | None: def prompt_integer(tariff_property: TariffPropertyStandard) -> float | None: + """Prompt for an integer-valued Arcadia tariff property.""" + default_value = tariff_property.get("property_value") if tariff_property["is_default"] else None default_value = int(default_value) if default_value else None @@ -137,10 +153,14 @@ def prompt_integer(tariff_property: TariffPropertyStandard) -> float | None: def prompt_demand(tariff_property: TariffPropertyStandard) -> float | None: + """Prompt for a demand-valued Arcadia tariff property.""" + return prompt_decimal(tariff_property) def _is_float(value: str) -> bool: + """Return whether a string can be parsed as a float.""" + try: _ = float(value) except ValueError: @@ -149,6 +169,8 @@ def _is_float(value: str) -> bool: def _is_int(value: str) -> bool: + """Return whether a string can be parsed as an integer.""" + try: _ = int(value) except ValueError: @@ -157,6 +179,8 @@ def _is_int(value: str) -> bool: def _get_property_msg(tariff_property: TariffPropertyStandard) -> str: + """Build a prompt label from an Arcadia property's display name and description.""" + title = tariff_property["display_name"] description = tariff_property["description"] return f"{title} ({description})" diff --git a/tariff_fetch/urdb/arcadia/rateutils.py b/tariff_fetch/urdb/arcadia/rateutils.py index 1dcd35c..ab72c65 100644 --- a/tariff_fetch/urdb/arcadia/rateutils.py +++ b/tariff_fetch/urdb/arcadia/rateutils.py @@ -1,3 +1,5 @@ +"""Shared Arcadia tariff filtering and time applicability helpers.""" + from collections.abc import Collection, Iterator from datetime import date, datetime from math import inf @@ -9,7 +11,7 @@ from tariff_fetch.arcadia.schema.timeofuse import Period, TimeOfUseExtended from tariff_fetch.urdb.arcadia.library import Library -from .exception import RateConversionError +from .exception import ConversionError, RateConversionError from .scenario import Scenario # ================================ @@ -23,6 +25,8 @@ def tariff_iter_rates_for_dt( library: Library, dt: datetime, ) -> Iterator[TariffRateExtended]: + """Yield all rate entries that apply for a scenario at a given datetime.""" + rates = tariff.get("rates", []) for rate in rates: if not rate_is_applied_to_scenario(rate, scenario, library): @@ -46,6 +50,8 @@ def rate_is_applied_to_scenario( scenario: Scenario, library: Library, ) -> bool: + """Return whether a rate matches scenario-level filters such as territory and charge class.""" + if not rate_is_applied_to_charge_classes(rate, scenario.charge_classes): return False if (territory := rate.get("territory")) is not None: @@ -58,12 +64,16 @@ def rate_is_applied_to_scenario( def rate_is_applied_to_charge_classes(rate: TariffRateExtended, charge_classes: Collection[RateChargeClass]) -> bool: + """Return whether a rate overlaps the requested Arcadia charge classes.""" + if (rate_charge_classes := rate.get("charge_class")) is None: return True return not set(rate_charge_classes).isdisjoint(charge_classes) def rate_is_applied_to_datetime(rate: TariffRateExtended, dt: datetime) -> bool: + """Return whether a rate is active at the given datetime.""" + if (season := rate.get("season")) and not season_is_datetime_within(season, dt): return False if (tou := rate.get("time_of_use")) and not time_of_use_is_datetime_within(tou, dt): @@ -72,6 +82,10 @@ def rate_is_applied_to_datetime(rate: TariffRateExtended, dt: datetime) -> bool: def rate_filter_bands(rate: TariffRateExtended, _scenario: Scenario, library: Library) -> list[TariffRateBand]: + """Return the supported rate bands whose applicability matches the current inputs.""" + + if rate.get("variable_limit_key") is not None: + raise RateConversionError(rate, "Rates with variable_limit_key are not supported") result: list[TariffRateBand] = [] for band in rate.get("rate_bands"): if band.get("has_demand_limit"): @@ -128,6 +142,8 @@ def rate_filter_bands(rate: TariffRateExtended, _scenario: Scenario, library: Li def rate_get_band_units(rate: TariffRateExtended) -> set[RateUnit]: + """Return the set of rate-unit values used by a rate's bands.""" + return {b["rate_unit"] for b in rate.get("rate_bands", [])} @@ -135,12 +151,18 @@ def rate_get_band_units(rate: TariffRateExtended) -> set[RateUnit]: # Rate Band # ================================ def band_consumption_upper_limit(band: TariffRateBand) -> float: + """Return a band's upper consumption limit, defaulting to infinity.""" + return band.get("consumption_upper_limit") or inf def rate_band_get_amount_at_datetime(band: TariffRateBand, library: Library, dt: datetime) -> float: + """Resolve the signed monetary amount for one rate band at a given datetime.""" + rate_id = band["tariff_rate_id"] rate = library.tariffs.get_rate(rate_id) + if rate.get("variable_factor_key") is not None: + raise RateConversionError(rate, "Rates with variable_factor_key are not supported") mp = -1 if band["is_credit"] else 1 if (variable_rate_key := rate.get("variable_rate_key")) is None: return band["rate_amount"] * mp @@ -156,6 +178,10 @@ def time_of_use_is_datetime_within( tou: TimeOfUseExtended, dt: datetime, ) -> bool: + """Return whether a datetime falls within a supported Arcadia TOU definition.""" + + if tou.get("calendar_id") is not None: + raise ConversionError("TOU calendar_id is not supported") season = tou.get("season") if season and not season_is_datetime_within(season, dt): return False @@ -172,6 +198,10 @@ def period_is_datetime_within( period: Period, dt: datetime, ) -> bool: + """Return whether a datetime falls within a TOU period's weekday and time bounds.""" + + if period.get("calendar_id") is not None: + raise ConversionError("TOU period calendar_id is not supported") return ( period["from_day_of_week"] <= dt.weekday() <= period["to_day_of_week"] and period["from_hour"] <= dt.hour <= period["to_hour"] @@ -188,6 +218,8 @@ def season_is_datetime_within( season: SeasonExtended, dt: datetime | date, ) -> bool: + """Return whether a date or datetime falls within a seasonal date window.""" + dt = dt.date() if isinstance(dt, datetime) else dt start_month, end_month = season["season_from_month"], season["season_to_month"] start_day, end_day = season["season_from_day"], season["season_to_day"] diff --git a/tariff_fetch/urdb/arcadia/scenario.py b/tariff_fetch/urdb/arcadia/scenario.py index b0848b4..eaee35d 100644 --- a/tariff_fetch/urdb/arcadia/scenario.py +++ b/tariff_fetch/urdb/arcadia/scenario.py @@ -1,3 +1,5 @@ +"""Scenario inputs that control one Arcadia-to-URDB conversion run.""" + from dataclasses import dataclass, field from typing import cast, get_args @@ -8,6 +10,8 @@ @dataclass(frozen=True) class Scenario: + """User-selected inputs for converting one Arcadia master tariff.""" + master_tariff_id: int year: int apply_percentages: bool diff --git a/tariff_fetch/urdb/arcadia/shared.py b/tariff_fetch/urdb/arcadia/shared.py index 1fb1e63..43b0e74 100644 --- a/tariff_fetch/urdb/arcadia/shared.py +++ b/tariff_fetch/urdb/arcadia/shared.py @@ -1,3 +1,5 @@ +"""Small shared helpers for Arcadia schedule sampling and date classification.""" + import calendar from datetime import date, datetime from functools import lru_cache @@ -13,6 +15,8 @@ def iter_sampled_datetimes( hour: int, day_filter: DayPredicate, ): + """Yield representative datetimes for all matching days in a month/hour bucket.""" + _, days = calendar.monthrange(year, month) for day in range(1, days + 1): @@ -32,10 +36,14 @@ def sample_datetime(year: int, month: int, day: int, hour: int) -> datetime: def is_weekday(dt: datetime) -> bool: + """Return whether a datetime falls on a weekday.""" + return dt.weekday() < 5 def is_weekend(dt: datetime) -> bool: + """Return whether a datetime falls on a weekend.""" + return dt.weekday() >= 5 @@ -44,6 +52,8 @@ def lookup_variable_rate( key: str, dt: datetime, ) -> float: + """Look up one variable Arcadia rate value at a specific datetime.""" + lookups = lookup_property_timeseries(api, key, dt.year) for row in lookups: if row["from_date_time"] <= dt <= (row["to_date_time"] or datetime.max): @@ -59,6 +69,8 @@ def lookup_variable_rate( @lru_cache(maxsize=256) def lookup_property_timeseries(api: ArcadiaSignalAPI, key: str, year: int): + """Fetch and cache one year of variable lookup rows for a property key.""" + return list( api.properties.lookups.iter_pages( key, diff --git a/tariff_fetch/urdb/arcadia/types.py b/tariff_fetch/urdb/arcadia/types.py index 17a8e39..6e1d08f 100644 --- a/tariff_fetch/urdb/arcadia/types.py +++ b/tariff_fetch/urdb/arcadia/types.py @@ -1,3 +1,5 @@ +"""Shared type aliases used by the Arcadia-to-URDB converter.""" + from collections.abc import Callable from datetime import datetime diff --git a/tests/test_arcadia_urdb_boundaries_build.py b/tests/test_arcadia_urdb_boundaries_build.py index 2a6bca6..bcdad98 100644 --- a/tests/test_arcadia_urdb_boundaries_build.py +++ b/tests/test_arcadia_urdb_boundaries_build.py @@ -1,9 +1,12 @@ from datetime import date, datetime from types import SimpleNamespace +import pytest + from tariff_fetch.urdb.arcadia import build as build_mod from tariff_fetch.urdb.arcadia import metadata as metadata_mod from tariff_fetch.urdb.arcadia import rateutils as ru +from tariff_fetch.urdb.arcadia.exception import ConversionError from tariff_fetch.urdb.arcadia.scenario import Scenario @@ -74,6 +77,31 @@ def test_time_of_use_is_datetime_within_applies_embedded_season(): assert ru.time_of_use_is_datetime_within(tou, datetime(2025, 7, 10, 12, 30)) is False # type: ignore[arg-type] +def test_time_of_use_is_datetime_within_rejects_calendar_id(): + tou = { + "calendar_id": 10, + "tou_periods": [], + } + + with pytest.raises(ConversionError, match="calendar_id"): + ru.time_of_use_is_datetime_within(tou, datetime(2025, 7, 10, 15, 30)) # type: ignore[arg-type] + + +def test_period_is_datetime_within_rejects_calendar_id(): + period = { + "from_day_of_week": 0, + "to_day_of_week": 6, + "from_hour": 0, + "to_hour": 23, + "from_minute": 0, + "to_minute": 59, + "calendar_id": 10, + } + + with pytest.raises(ConversionError, match="calendar_id"): + ru.period_is_datetime_within(period, datetime(2025, 7, 10, 15, 30)) # type: ignore[arg-type] + + def test_build_urdb_merges_converter_chunks(monkeypatch): scenario = Scenario(123, 2025, apply_percentages=True, charge_classes={"SUPPLY"}) diff --git a/tests/test_arcadia_urdb_fixedcharge.py b/tests/test_arcadia_urdb_fixedcharge.py index 14b48a9..ba8d0a6 100644 --- a/tests/test_arcadia_urdb_fixedcharge.py +++ b/tests/test_arcadia_urdb_fixedcharge.py @@ -2,6 +2,7 @@ import pytest +from tariff_fetch.urdb.arcadia.exception import RateConversionError from tariff_fetch.urdb.arcadia.fixedcharge import get_rate_fixed_charge_at_dt, normalize_fixed_charge_amount from tests.arcadia_urdb_fixtures import make_band, make_fixed_rate @@ -49,3 +50,15 @@ def test_get_rate_fixed_charge_converts_daily_to_monthly(monkeypatch): def test_normalize_fixed_charge_amount_rejects_unsupported_period(): with pytest.raises(ValueError, match="Unsupported fixed charge period"): normalize_fixed_charge_amount(1.0, "YEARLY", datetime(2025, 1, 1, 0, 30)) + + +def test_get_rate_fixed_charge_rejects_quantity_key(): + rate = make_fixed_rate(quantity_key="billingMeter") + + with pytest.raises(RateConversionError, match="quantity_key"): + get_rate_fixed_charge_at_dt( + scenario=None, # pyright: ignore[reportArgumentType] + library=None, # pyright: ignore[reportArgumentType] + rate=rate, # pyright: ignore[reportArgumentType] + dt=datetime(2025, 1, 1, 0, 30), + ) diff --git a/tests/test_arcadia_urdb_rateutils.py b/tests/test_arcadia_urdb_rateutils.py index d9dacb6..5ce78f9 100644 --- a/tests/test_arcadia_urdb_rateutils.py +++ b/tests/test_arcadia_urdb_rateutils.py @@ -77,6 +77,16 @@ def test_rate_filter_bands_rejects_unsupported_operator(): ru.rate_filter_bands(rate, Scenario(1, 2025, False), library) # type: ignore[arg-type] +def test_rate_filter_bands_rejects_variable_limit_key(): + rate = { + "variable_limit_key": "demandMultiplierTiers", + "rate_bands": [make_band()], + } + + with pytest.raises(RateConversionError, match="variable_limit_key"): + ru.rate_filter_bands(rate, Scenario(1, 2025, False), StubLibrary()) # type: ignore[arg-type] + + def test_rate_is_applied_to_scenario_filters_territory(): rate = { "charge_class": ["SUPPLY"], @@ -138,3 +148,28 @@ def test_get_raw_bands_at_datetime_skips_percentage_when_disabled(monkeypatch): ) assert result == [(inf, 10.0)] + + +def test_rate_band_get_amount_at_datetime_rejects_variable_factor_key(): + band = make_band(tariff_rate_id=7, rate_amount=5.0) + rate = make_consumption_rate(tariff_rate_id=7, variable_factor_key="billingPeriodProrationFactor") + library = SimpleNamespace(tariffs=SimpleNamespace(get_rate=lambda rate_id: rate)) + + with pytest.raises(RateConversionError, match="variable_factor_key"): + ru.rate_band_get_amount_at_datetime( + band, # type: ignore[arg-type] + library, # type: ignore[arg-type] + datetime(2025, 1, 1, 0, 30), + ) + + +def test_get_rate_consumption_bands_rejects_quantity_key(): + rate = make_consumption_rate(quantity_key="billingMeter") + + with pytest.raises(RateConversionError, match="quantity_key"): + es.get_rate_consumption_bands_at_datetime( + rate=rate, # type: ignore[arg-type] + scenario=None, # pyright: ignore[reportArgumentType] + library=None, # pyright: ignore[reportArgumentType] + dt=datetime(2025, 1, 1, 0, 30), + ) From 4a50f1c0062a373b6a4c193a17c07d6a7bbb5e4f Mon Sep 17 00:00:00 2001 From: Natalia Borovskikh Date: Thu, 12 Mar 2026 14:18:26 +0000 Subject: [PATCH 15/24] update conversion logic --- docs/providers/arcadia/urdb-converter.md | 10 +++ tariff_fetch/urdb/arcadia/build.py | 1 - tariff_fetch/urdb/arcadia/fixedcharge.py | 2 +- tariff_fetch/urdb/arcadia/library.py | 5 +- tariff_fetch/urdb/arcadia/rateutils.py | 8 +- tariff_fetch/urdb/arcadia/shared.py | 12 ++- tests/arcadia_urdb_fixtures.py | 96 +++++++++++++++++++++ tests/test_arcadia_urdb_boundaries_build.py | 55 +++++++++++- 8 files changed, 183 insertions(+), 6 deletions(-) create mode 100644 tests/arcadia_urdb_fixtures.py diff --git a/docs/providers/arcadia/urdb-converter.md b/docs/providers/arcadia/urdb-converter.md index acb0198..91e0086 100644 --- a/docs/providers/arcadia/urdb-converter.md +++ b/docs/providers/arcadia/urdb-converter.md @@ -93,6 +93,16 @@ Some Arcadia tariffs include percentage-based adjustments. These can be included This makes it easier to compare a cleaner base energy schedule against a schedule that includes percentage adders. +Percentage handling should be understood as an approximation. The converter treats supported percentage rates as linear modifiers on matching energy charges. That is often reasonable for simple tariffs, but it is not the same as reproducing Arcadia's full bill-calculation logic in every case. + +The approximation is most trustworthy when: + +- the percentage rate is a straightforward adder or surcharge +- the affected charges are ordinary energy charges +- the intended percentage base lines up well with the selected charge classes + +It is less trustworthy when the percentage is really meant to apply to a bill subtotal with exclusions, ordering rules, caps, floors, taxes, or other bill-level logic. + ## Debug output For troubleshooting, the converter saves fetched Arcadia data under `outputs/arcadia_library/` by default. diff --git a/tariff_fetch/urdb/arcadia/build.py b/tariff_fetch/urdb/arcadia/build.py index c6f895f..d893e02 100644 --- a/tariff_fetch/urdb/arcadia/build.py +++ b/tariff_fetch/urdb/arcadia/build.py @@ -1,6 +1,5 @@ """Build URDB-style output from Arcadia tariff data for a conversion scenario.""" -from dataclasses import dataclass from typing import cast import questionary diff --git a/tariff_fetch/urdb/arcadia/fixedcharge.py b/tariff_fetch/urdb/arcadia/fixedcharge.py index 8e6d101..c28237e 100644 --- a/tariff_fetch/urdb/arcadia/fixedcharge.py +++ b/tariff_fetch/urdb/arcadia/fixedcharge.py @@ -59,7 +59,7 @@ def get_rate_fixed_charge_at_dt(scenario: Scenario, library: Library, rate: Tari raise RateConversionError(rate, "Fixed price rate bands units should be COST_PER_UNIT") if (charge_period := rate["charge_period"]) not in {"MONTHLY", "DAILY"}: raise RateConversionError(rate, f"Fixed charges should be monthly or daily (got {charge_period})") - if len(band_rate_units) > 1: + if len(bands) > 1: raise RateConversionError(rate, "More than one applicable band for percentage rate") band = bands[0] if band["has_consumption_limit"]: diff --git a/tariff_fetch/urdb/arcadia/library.py b/tariff_fetch/urdb/arcadia/library.py index 21c9e28..17ef704 100644 --- a/tariff_fetch/urdb/arcadia/library.py +++ b/tariff_fetch/urdb/arcadia/library.py @@ -22,6 +22,7 @@ prompt_integer, prompt_string, ) +from .shared import as_naive_datetime PropertyValue = str | list[str] | bool | date | float | int _DEFAULT_DEBUG_ROOT = Path("./outputs/arcadia_library") @@ -188,7 +189,9 @@ def lookup(self, key: str, dt: datetime) -> float: self.debug_store.save_lookups(key, dt.year, lookups) for row in lookups: - if row["from_date_time"] <= dt <= (row["to_date_time"] or datetime.max): + if as_naive_datetime(row["from_date_time"]) <= as_naive_datetime(dt) <= as_naive_datetime( + row["to_date_time"] or datetime.max + ): if (value := row["actual_value"]) is not None: return value if (value := row["best_value"]) is not None: diff --git a/tariff_fetch/urdb/arcadia/rateutils.py b/tariff_fetch/urdb/arcadia/rateutils.py index ab72c65..2893aed 100644 --- a/tariff_fetch/urdb/arcadia/rateutils.py +++ b/tariff_fetch/urdb/arcadia/rateutils.py @@ -13,6 +13,7 @@ from .exception import ConversionError, RateConversionError from .scenario import Scenario +from .shared import as_naive_datetime # ================================ # Tariff @@ -74,6 +75,11 @@ def rate_is_applied_to_charge_classes(rate: TariffRateExtended, charge_classes: def rate_is_applied_to_datetime(rate: TariffRateExtended, dt: datetime) -> bool: """Return whether a rate is active at the given datetime.""" + dt_naive = as_naive_datetime(dt) + if (from_dt := rate.get("from_date_time")) is not None and dt_naive < as_naive_datetime(from_dt): + return False + if (to_dt := rate.get("to_date_time")) is not None and dt_naive >= as_naive_datetime(to_dt): + return False if (season := rate.get("season")) and not season_is_datetime_within(season, dt): return False if (tou := rate.get("time_of_use")) and not time_of_use_is_datetime_within(tou, dt): @@ -121,7 +127,7 @@ def rate_filter_bands(rate: TariffRateExtended, _scenario: Scenario, library: Li if applicability_value not in library.get_property(applicability_key, "CHOICE"): continue case "STRING": - if applicability_value != library.get_property(applicability_value, "STRING"): + if applicability_value != library.get_property(applicability_key, "STRING"): continue case "BOOLEAN": match applicability_value: diff --git a/tariff_fetch/urdb/arcadia/shared.py b/tariff_fetch/urdb/arcadia/shared.py index 43b0e74..69fc8b0 100644 --- a/tariff_fetch/urdb/arcadia/shared.py +++ b/tariff_fetch/urdb/arcadia/shared.py @@ -9,6 +9,12 @@ from .types import DayPredicate +def as_naive_datetime(dt: datetime) -> datetime: + """Drop timezone metadata so Arcadia timestamps compare as local wall-clock datetimes.""" + + return dt.replace(tzinfo=None) + + def iter_sampled_datetimes( year: int, month: int, @@ -56,7 +62,11 @@ def lookup_variable_rate( lookups = lookup_property_timeseries(api, key, dt.year) for row in lookups: - if row["from_date_time"] <= dt <= (row["to_date_time"] or datetime.max): + if ( + as_naive_datetime(row["from_date_time"]) + <= as_naive_datetime(dt) + <= as_naive_datetime(row["to_date_time"] or datetime.max) + ): if (value := row["actual_value"]) is not None: return value if (value := row["best_value"]) is not None: diff --git a/tests/arcadia_urdb_fixtures.py b/tests/arcadia_urdb_fixtures.py new file mode 100644 index 0000000..2b315b0 --- /dev/null +++ b/tests/arcadia_urdb_fixtures.py @@ -0,0 +1,96 @@ +from dataclasses import dataclass +from math import inf +from types import SimpleNamespace +from typing import Any + + +def make_band(**overrides: Any) -> dict[str, Any]: + band = { + "tariff_rate_id": 1, + "rate_unit": "COST_PER_UNIT", + "rate_amount": 1.0, + "is_credit": False, + "has_consumption_limit": False, + "consumption_upper_limit": None, + "has_demand_limit": False, + "demand_upper_limit": None, + "has_property_limit": False, + "property_upper_limit": None, + "calculation_factor": None, + "applicability_formula": None, + } + band.update(overrides) + return band + + +def make_rate(**overrides: Any) -> dict[str, Any]: + rate = { + "charge_type": "FIXED_PRICE", + "transaction_type": "BUY", + "charge_period": "MONTHLY", + "charge_class": ["SUPPLY"], + "tariff_rate_id": 1, + "rate_name": "Customer Charge", + "rate_bands": [make_band()], + } + rate.update(overrides) + return rate + + +def make_fixed_rate(**overrides: Any) -> dict[str, Any]: + return make_rate(**overrides) + + +def make_consumption_rate(**overrides: Any) -> dict[str, Any]: + rate = make_rate( + charge_type="CONSUMPTION_BASED", + rate_name="Energy Charge", + rate_bands=[make_band(consumption_upper_limit=inf)], + ) + rate.update(overrides) + return rate + + +def make_percentage_rate(**overrides: Any) -> dict[str, Any]: + rate = make_rate( + charge_type="FIXED_PRICE", + rate_name="Percentage Charge", + rate_bands=[make_band(rate_unit="PERCENTAGE")], + ) + rate.update(overrides) + return rate + + +def make_library_with_tariff(rate: dict[str, Any]) -> SimpleNamespace: + tariff = {"rates": []} + return SimpleNamespace( + tariffs=SimpleNamespace(get_tariff_at_date=lambda master_tariff_id, dt: tariff, get_rate=lambda rate_id: rate), + variables=None, + ) + + +@dataclass +class StubTariffLibrary: + properties: dict[str, dict[str, object]] + + def get_property(self, key: str) -> dict[str, object]: + return self.properties[key] + + +class StubLibrary: + def __init__( + self, + *, + properties: dict[str, object] | None = None, + tariff_properties: dict[str, dict[str, object]] | None = None, + ) -> None: + self._properties = properties or {} + self.tariffs = StubTariffLibrary(tariff_properties or {}) + + def get_property(self, key: str, data_type: str) -> object: + return self._properties[key] + + def get_choice_property_as_ints(self, key: str) -> list[int]: + values = self._properties[key] + assert isinstance(values, list) + return [int(value) for value in values] diff --git a/tests/test_arcadia_urdb_boundaries_build.py b/tests/test_arcadia_urdb_boundaries_build.py index bcdad98..1421cea 100644 --- a/tests/test_arcadia_urdb_boundaries_build.py +++ b/tests/test_arcadia_urdb_boundaries_build.py @@ -1,4 +1,4 @@ -from datetime import date, datetime +from datetime import date, datetime, timezone from types import SimpleNamespace import pytest @@ -77,6 +77,59 @@ def test_time_of_use_is_datetime_within_applies_embedded_season(): assert ru.time_of_use_is_datetime_within(tou, datetime(2025, 7, 10, 12, 30)) is False # type: ignore[arg-type] +def test_rate_is_applied_to_datetime_respects_rate_level_effective_window(): + rate = { + "from_date_time": datetime(2025, 3, 1, 0, 0), + "to_date_time": datetime(2025, 6, 1, 0, 0), + } + + assert ru.rate_is_applied_to_datetime(rate, datetime(2025, 2, 28, 23, 59)) is False # type: ignore[arg-type] + assert ru.rate_is_applied_to_datetime(rate, datetime(2025, 3, 1, 0, 0)) is True # type: ignore[arg-type] + assert ru.rate_is_applied_to_datetime(rate, datetime(2025, 5, 31, 23, 59)) is True # type: ignore[arg-type] + assert ru.rate_is_applied_to_datetime(rate, datetime(2025, 6, 1, 0, 0)) is False # type: ignore[arg-type] + + +def test_rate_is_applied_to_datetime_combines_effective_window_with_season_and_tou(): + rate = { + "from_date_time": datetime(2025, 6, 1, 0, 0), + "to_date_time": datetime(2025, 9, 1, 0, 0), + "season": { + "season_from_month": 6, + "season_from_day": 1, + "season_to_month": 9, + "season_to_day": 1, + }, + "time_of_use": { + "tou_periods": [ + { + "from_day_of_week": 0, + "to_day_of_week": 6, + "from_hour": 14, + "to_hour": 18, + "from_minute": 0, + "to_minute": 59, + } + ] + }, + } + + assert ru.rate_is_applied_to_datetime(rate, datetime(2025, 7, 10, 15, 30)) is True # type: ignore[arg-type] + assert ru.rate_is_applied_to_datetime(rate, datetime(2025, 5, 31, 15, 30)) is False # type: ignore[arg-type] + assert ru.rate_is_applied_to_datetime(rate, datetime(2025, 7, 10, 12, 30)) is False # type: ignore[arg-type] + assert ru.rate_is_applied_to_datetime(rate, datetime(2025, 9, 1, 0, 0)) is False # type: ignore[arg-type] + + +def test_rate_is_applied_to_datetime_handles_offset_aware_rate_window(): + rate = { + "from_date_time": datetime(2025, 3, 1, 0, 0, tzinfo=timezone.utc), + "to_date_time": datetime(2025, 6, 1, 0, 0, tzinfo=timezone.utc), + } + + assert ru.rate_is_applied_to_datetime(rate, datetime(2025, 2, 28, 23, 59)) is False # type: ignore[arg-type] + assert ru.rate_is_applied_to_datetime(rate, datetime(2025, 3, 1, 0, 0)) is True # type: ignore[arg-type] + assert ru.rate_is_applied_to_datetime(rate, datetime(2025, 6, 1, 0, 0)) is False # type: ignore[arg-type] + + def test_time_of_use_is_datetime_within_rejects_calendar_id(): tou = { "calendar_id": 10, From abaefff790001062870f8999e1d0e49e4df98aeb Mon Sep 17 00:00:00 2001 From: Natalia Borovskikh Date: Thu, 12 Mar 2026 14:21:46 +0000 Subject: [PATCH 16/24] remove per-rate logging --- tariff_fetch/urdb/arcadia/fixedcharge.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tariff_fetch/urdb/arcadia/fixedcharge.py b/tariff_fetch/urdb/arcadia/fixedcharge.py index c28237e..764208e 100644 --- a/tariff_fetch/urdb/arcadia/fixedcharge.py +++ b/tariff_fetch/urdb/arcadia/fixedcharge.py @@ -13,8 +13,6 @@ from .library import Library from .scenario import Scenario -_LOGGED: set[int] = set() - def build_fixed_charge(scenario: Scenario, library: Library) -> URDBRate: """Build the URDB fixed-charge fields for a scenario.""" @@ -81,9 +79,6 @@ def get_rate_fixed_charge_at_dt(scenario: Scenario, library: Library, rate: Tari rate_amount = ru.rate_band_get_amount_at_datetime(band, library, dt) rate_amount = normalize_fixed_charge_amount(rate_amount, charge_period, dt) - if rate["tariff_rate_id"] not in _LOGGED: - print(f"Applied fixed charge: {rate['rate_name']} ({rate['tariff_rate_id']}) ({rate_amount})") - _LOGGED.add(rate["tariff_rate_id"]) return rate_amount From d8f78722c0852cdb7543ed410a8b1157482e2028 Mon Sep 17 00:00:00 2001 From: Natalia Borovskikh Date: Thu, 12 Mar 2026 14:27:40 +0000 Subject: [PATCH 17/24] fail-fast on variable_rate_sub_key --- tariff_fetch/urdb/arcadia/rateutils.py | 2 ++ tests/test_arcadia_urdb_rateutils.py | 13 +++++++++++++ 2 files changed, 15 insertions(+) diff --git a/tariff_fetch/urdb/arcadia/rateutils.py b/tariff_fetch/urdb/arcadia/rateutils.py index 2893aed..ea4d09b 100644 --- a/tariff_fetch/urdb/arcadia/rateutils.py +++ b/tariff_fetch/urdb/arcadia/rateutils.py @@ -167,6 +167,8 @@ def rate_band_get_amount_at_datetime(band: TariffRateBand, library: Library, dt: rate_id = band["tariff_rate_id"] rate = library.tariffs.get_rate(rate_id) + if rate.get("variable_rate_sub_key") is not None: + raise RateConversionError(rate, "Rates with variable_rate_sub_key are not supported") if rate.get("variable_factor_key") is not None: raise RateConversionError(rate, "Rates with variable_factor_key are not supported") mp = -1 if band["is_credit"] else 1 diff --git a/tests/test_arcadia_urdb_rateutils.py b/tests/test_arcadia_urdb_rateutils.py index 5ce78f9..5dad26f 100644 --- a/tests/test_arcadia_urdb_rateutils.py +++ b/tests/test_arcadia_urdb_rateutils.py @@ -163,6 +163,19 @@ def test_rate_band_get_amount_at_datetime_rejects_variable_factor_key(): ) +def test_rate_band_get_amount_at_datetime_rejects_variable_rate_sub_key(): + band = make_band(tariff_rate_id=8, rate_amount=5.0) + rate = make_consumption_rate(tariff_rate_id=8, variable_rate_sub_key="zoneA") + library = SimpleNamespace(tariffs=SimpleNamespace(get_rate=lambda rate_id: rate)) + + with pytest.raises(RateConversionError, match="variable_rate_sub_key"): + ru.rate_band_get_amount_at_datetime( + band, # type: ignore[arg-type] + library, # type: ignore[arg-type] + datetime(2025, 1, 1, 0, 30), + ) + + def test_get_rate_consumption_bands_rejects_quantity_key(): rate = make_consumption_rate(quantity_key="billingMeter") From c7c151c91bd35eca6b9f28fd5597720f24dd8c3d Mon Sep 17 00:00:00 2001 From: Natalia Borovskikh Date: Thu, 12 Mar 2026 14:28:07 +0000 Subject: [PATCH 18/24] update version' --- pyproject.toml | 2 +- uv.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d8d9471..50a1683 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "tariff_fetch" -version = "0.4.6" +version = "0.4.7" description = "A CLI tool and a python library to simplify downloading electric and gas utility tariff data." authors = [{ name = "Switchbox", email = "hello@switch.box" }] readme = "README.md" diff --git a/uv.lock b/uv.lock index 9766cdb..2fdb5dc 100644 --- a/uv.lock +++ b/uv.lock @@ -1480,7 +1480,7 @@ wheels = [ [[package]] name = "tariff-fetch" -version = "0.4.6" +version = "0.4.7" source = { editable = "." } dependencies = [ { name = "fastexcel" }, From e235a527ac2aaa981c3245192e9fd140da183a2e Mon Sep 17 00:00:00 2001 From: Natalia Borovskikh Date: Thu, 12 Mar 2026 14:30:38 +0000 Subject: [PATCH 19/24] Subclass tariff not found errors from a proper base --- tariff_fetch/urdb/arcadia/exception.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tariff_fetch/urdb/arcadia/exception.py b/tariff_fetch/urdb/arcadia/exception.py index 0ac0443..a91fab0 100644 --- a/tariff_fetch/urdb/arcadia/exception.py +++ b/tariff_fetch/urdb/arcadia/exception.py @@ -37,7 +37,7 @@ class TariffNotFoundError(ConversionError): @final -class TariffNotFoundById(ConversionError): +class TariffNotFoundById(TariffNotFoundError): """Raised when a tariff version cannot be found by Arcadia tariff id.""" def __init__(self, tariff_id: int) -> None: @@ -50,7 +50,7 @@ def __str__(self) -> str: @final -class TariffNotFoundByDate(ConversionError): +class TariffNotFoundByDate(TariffNotFoundError): """Raised when no tariff version is effective for a requested date.""" def __init__(self, master_tariff_id: int, dt: date) -> None: From 8f804d4e9b98c1a462b149be7f975cea5073a72f Mon Sep 17 00:00:00 2001 From: Natalia Borovskikh Date: Thu, 12 Mar 2026 15:14:47 +0000 Subject: [PATCH 20/24] Add cli-arcadia-urdb --- README.md | 16 ++++ pyproject.toml | 3 +- tariff_fetch/cli_arcadia_urdb.py | 107 +++++++++++++++++++++++ tariff_fetch/urdb/arcadia/build.py | 7 ++ tariff_fetch/urdb/arcadia/exception.py | 13 +++ tariff_fetch/urdb/arcadia/fixedcharge.py | 2 - tariff_fetch/urdb/arcadia/library.py | 43 ++++++--- tariff_fetch/urdb/arcadia/rateutils.py | 21 +++-- tests/test_arcadia_urdb_fixedcharge.py | 22 +++++ tests/test_arcadia_urdb_rateutils.py | 103 +++++++++++++++++++++- uv.lock | 2 +- 11 files changed, 316 insertions(+), 23 deletions(-) create mode 100644 tariff_fetch/cli_arcadia_urdb.py diff --git a/README.md b/README.md index a7fc57f..f701a21 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,7 @@ With uv: ```bash uv run tariff-fetch [OPTIONS] uv run tariff-fetch-gas [OPTIONS] +uv run tariff-fetch-arcadia-urdb MASTER_TARIFF_ID YEAR [OPTIONS] ``` With Just: @@ -116,3 +117,18 @@ uv run tariff-fetch.cli \ The CLI suggests filenames like `outputs/openei_Utility_sector_detail-0_2024-03-18.json` before writing each file so you can accept or override them. + +## Direct Arcadia to URDB Conversion + +For direct conversion of a single Arcadia master tariff to URDB JSON: + +```bash +uv run tariff-fetch-arcadia-urdb 522 2025 +``` + +Useful options: + +- `--output` / `-o`: output file path +- `--apply-percentages` / `--no-apply-percentages` +- `--charge-class`: repeat to include multiple charge classes +- `--force` / `-f`: overwrite an existing output file diff --git a/pyproject.toml b/pyproject.toml index 50a1683..d2e6e7d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "tariff_fetch" -version = "0.4.7" +version = "0.4.8" description = "A CLI tool and a python library to simplify downloading electric and gas utility tariff data." authors = [{ name = "Switchbox", email = "hello@switch.box" }] readme = "README.md" @@ -39,6 +39,7 @@ Documentation = "https://switchbox-data.github.io/tariff_fetch/" [project.scripts] tariff-fetch = "tariff_fetch.cli:main_cli" tariff-fetch-gas = "tariff_fetch.cli_gas:main_cli" +tariff-fetch-arcadia-urdb = "tariff_fetch.cli_arcadia_urdb:main_cli" [dependency-groups] dev = [ diff --git a/tariff_fetch/cli_arcadia_urdb.py b/tariff_fetch/cli_arcadia_urdb.py new file mode 100644 index 0000000..bf2f227 --- /dev/null +++ b/tariff_fetch/cli_arcadia_urdb.py @@ -0,0 +1,107 @@ +"""Direct CLI for converting one Arcadia master tariff to URDB.""" + +import json +import logging +import os +from pathlib import Path +from typing import Annotated, cast, get_args + +import typer +from dotenv import load_dotenv +from rich.logging import RichHandler + +from tariff_fetch.arcadia.api import ArcadiaSignalAPI +from tariff_fetch.arcadia.schema.common import RateChargeClass +from tariff_fetch.urdb.arcadia.build import build_urdb +from tariff_fetch.urdb.arcadia.scenario import Scenario + +from ._cli import console + +FORMAT = "%(message)s" +logging.basicConfig(level="NOTSET", format=FORMAT, datefmt="[%X]", handlers=[RichHandler()]) + +DEFAULT_CHARGE_CLASSES: tuple[RateChargeClass, ...] = ( + "DISTRIBUTION", + "SUPPLY", + "TRANSMISSION", + "OTHER", + "CONTRACTED", +) +ALL_CHARGE_CLASSES = cast(tuple[RateChargeClass, ...], get_args(RateChargeClass)) + + +def main( + master_tariff_id: Annotated[int, typer.Argument(help="Arcadia master tariff id to convert")], + year: Annotated[int, typer.Argument(help="Calendar year to convert")], + output: Annotated[ + Path | None, + typer.Option("--output", "-o", help="Path to write the converted URDB JSON"), + ] = None, + apply_percentages: Annotated[ + bool, + typer.Option("--apply-percentages/--no-apply-percentages", help="Apply supported percentage rates"), + ] = False, + charge_classes: Annotated[ + list[str] | None, + typer.Option("--charge-class", help="Arcadia charge class to include; repeat to include multiple"), + ] = None, + force: Annotated[ + bool, + typer.Option("--force", "-f", help="Overwrite the output file if it already exists"), + ] = False, +) -> None: + """Convert one Arcadia master tariff to a URDB JSON file.""" + + _ = load_dotenv() + if not os.getenv("ARCADIA_APP_ID"): + console.print("[b]ARCADIA_APP_ID[/] environment variable is not set.") + if not os.getenv("ARCADIA_APP_KEY"): + console.print("[b]ARCADIA_APP_KEY[/] environment variable is not set.") + if not (os.getenv("ARCADIA_APP_ID") and os.getenv("ARCADIA_APP_KEY")): + raise typer.Exit(code=1) + + output_path = output or Path(f"./outputs/arcadia_urdb_{master_tariff_id}_{year}.json") + if output_path.exists() and not force: + console.print(f"[red]Output file already exists:[/] {output_path}") + console.print("Pass [b]--force[/] to overwrite it.") + raise typer.Exit(code=1) + + scenario_charge_classes = _parse_charge_classes(charge_classes) + scenario = Scenario( + master_tariff_id=master_tariff_id, + year=year, + apply_percentages=apply_percentages, + charge_classes=scenario_charge_classes, + ) + api = ArcadiaSignalAPI() + + console.print("Converting Arcadia tariff to URDB...") + result = build_urdb(api, scenario) + + output_path.parent.mkdir(parents=True, exist_ok=True) + _ = output_path.write_text(json.dumps(result, indent=2)) + console.print(f"Wrote URDB tariff to [blue]{output_path}[/]") + + +def main_cli() -> None: + """Run the direct Arcadia-to-URDB CLI.""" + + typer.run(main) + + +def _parse_charge_classes(charge_classes: list[str] | None) -> set[RateChargeClass]: + if charge_classes is None: + return set(DEFAULT_CHARGE_CLASSES) + + normalized = [charge_class.strip().upper() for charge_class in charge_classes] + invalid = sorted(set(normalized) - set(ALL_CHARGE_CLASSES)) + if invalid: + allowed = ", ".join(ALL_CHARGE_CLASSES) + console.print(f"[red]Invalid charge classes:[/] {', '.join(invalid)}") + console.print(f"Allowed values: {allowed}") + raise typer.Exit(code=1) + return {cast(RateChargeClass, charge_class) for charge_class in normalized} + + +if __name__ == "__main__": + main_cli() diff --git a/tariff_fetch/urdb/arcadia/build.py b/tariff_fetch/urdb/arcadia/build.py index d893e02..114da06 100644 --- a/tariff_fetch/urdb/arcadia/build.py +++ b/tariff_fetch/urdb/arcadia/build.py @@ -1,5 +1,6 @@ """Build URDB-style output from Arcadia tariff data for a conversion scenario.""" +import logging from typing import cast import questionary @@ -13,6 +14,8 @@ from .metadata import build_metadata from .scenario import Scenario +logger = logging.getLogger(__name__) + def build_urdb(api: ArcadiaSignalAPI, scenario: Scenario) -> URDBRate: """Build a URDB record by combining energy, fixed-charge, and metadata chunks.""" @@ -33,6 +36,10 @@ def build_urdb(api: ArcadiaSignalAPI, scenario: Scenario) -> URDBRate: except Exception as e: metadata = _confirm_proceed(e, "metadata") + if hasattr(library, "iter_issues"): + for issue in library.iter_issues(): + logger.warning(issue) + return {**energy_schedule, **fixed_charge, **metadata} diff --git a/tariff_fetch/urdb/arcadia/exception.py b/tariff_fetch/urdb/arcadia/exception.py index a91fab0..ada64f4 100644 --- a/tariff_fetch/urdb/arcadia/exception.py +++ b/tariff_fetch/urdb/arcadia/exception.py @@ -36,6 +36,19 @@ class TariffNotFoundError(ConversionError): """Base exception for tariff lookup misses.""" +@final +class TariffAccessDenied(ConversionError): + """Raised when Arcadia denies access to a tariff id.""" + + def __init__(self, tariff_id: int) -> None: + self.tariff_id = tariff_id + super().__init__() + + @override + def __str__(self) -> str: + return f"Access denied for tariff id={self.tariff_id}" + + @final class TariffNotFoundById(TariffNotFoundError): """Raised when a tariff version cannot be found by Arcadia tariff id.""" diff --git a/tariff_fetch/urdb/arcadia/fixedcharge.py b/tariff_fetch/urdb/arcadia/fixedcharge.py index 764208e..dec19f7 100644 --- a/tariff_fetch/urdb/arcadia/fixedcharge.py +++ b/tariff_fetch/urdb/arcadia/fixedcharge.py @@ -72,8 +72,6 @@ def get_rate_fixed_charge_at_dt(scenario: Scenario, library: Library, rate: Tari raise RateConversionError(rate, "Fixed rate bands cannot have has_property_limit==true") if band.get("property_upper_limit") is not None: raise RateConversionError(rate, "Fixed rate bands cannot have property_upper_limit") - if band.get("calculation_factor") is not None: - raise RateConversionError(rate, "Fixed rate bands cannot have calculation_factor") if band.get("applicability_formula") is not None: raise RateConversionError(rate, "Fixed rate bands cannot have applicability_formula") diff --git a/tariff_fetch/urdb/arcadia/library.py b/tariff_fetch/urdb/arcadia/library.py index 17ef704..949f2f7 100644 --- a/tariff_fetch/urdb/arcadia/library.py +++ b/tariff_fetch/urdb/arcadia/library.py @@ -5,12 +5,13 @@ from pathlib import Path from typing import Literal, final, overload +from requests import HTTPError from tariff_fetch.arcadia.api import ArcadiaSignalAPI from tariff_fetch.arcadia.schema.lookup import Lookup from tariff_fetch.arcadia.schema.tariff import TariffExtended from tariff_fetch.arcadia.schema.tariffproperty import TariffPropertyPrunedDataType, TariffPropertyStandard from tariff_fetch.arcadia.schema.tariffrate import TariffRateExtended -from tariff_fetch.urdb.arcadia.exception import TariffNotFoundByDate, TariffNotFoundById +from tariff_fetch.urdb.arcadia.exception import TariffAccessDenied, TariffNotFoundByDate, TariffNotFoundById from .exception import ConversionError from .prompts import ( @@ -82,6 +83,7 @@ def __init__(self, api: ArcadiaSignalAPI, debug_store: LibraryDebugStore) -> Non self.api = api self.debug_store = debug_store self.tariffs: list[TariffExtended] = [] + self._access_denied_tariff_ids: set[int] = set() def get_tariff_at_date(self, master_tariff_id: int, dt: date) -> TariffExtended: """Return the tariff version effective for a master tariff on a given date.""" @@ -98,6 +100,8 @@ def get_tariff(self, tariff_id: int) -> TariffExtended: if (found := self._find_tariff(tariff_id)) is not None: return found + if tariff_id in self._access_denied_tariff_ids: + raise TariffAccessDenied(tariff_id) tariff = self._fetch_tariff(tariff_id) self._remember(tariff) return tariff @@ -138,17 +142,23 @@ def _remember(self, tariff: TariffExtended) -> None: def _fetch_tariff(self, tariff_id: int) -> TariffExtended: """Fetch one tariff version from Arcadia by tariff id.""" - tariffs = list( - self.api.tariffs.iter_pages( - fields="ext", - search=str(tariff_id), - search_on=["tariffId"], - starts_with=True, - ends_with=True, - populate_properties=True, - populate_rates=True, + try: + tariffs = list( + self.api.tariffs.iter_pages( + fields="ext", + search=str(tariff_id), + search_on=["tariffId"], + starts_with=True, + ends_with=True, + populate_properties=True, + populate_rates=True, + ) ) - ) + except HTTPError as e: + if e.response is not None and e.response.status_code == 403: + self._access_denied_tariff_ids.add(tariff_id) + raise TariffAccessDenied(tariff_id) from e + raise if len(tariffs) > 1: raise RuntimeError(f"More than one tariff found for id={tariff_id}") if not tariffs: @@ -228,12 +238,23 @@ def __init__( self.tariffs = TariffLibrary(api, self.debug_store) self.variables = VariablePropertyLibrary(api, self.debug_store) self._properies: dict[str, PropertyValue] = properties or {} + self._issues: dict[tuple[object, ...], str] = {} def has_property(self, key: str) -> bool: """Return whether a property value has already been supplied or cached.""" return key in self._properies + def record_issue(self, key: tuple[object, ...], message: str) -> None: + """Record a deduplicated conversion issue for this conversion run.""" + + _ = self._issues.setdefault(key, message) + + def iter_issues(self) -> list[str]: + """Return the unique conversion issues recorded during this conversion run.""" + + return list(self._issues.values()) + def get_choice_property_as_ints(self, key: str) -> list[int]: """Return a CHOICE property as integer ids.""" diff --git a/tariff_fetch/urdb/arcadia/rateutils.py b/tariff_fetch/urdb/arcadia/rateutils.py index ea4d09b..0198a34 100644 --- a/tariff_fetch/urdb/arcadia/rateutils.py +++ b/tariff_fetch/urdb/arcadia/rateutils.py @@ -11,7 +11,7 @@ from tariff_fetch.arcadia.schema.timeofuse import Period, TimeOfUseExtended from tariff_fetch.urdb.arcadia.library import Library -from .exception import ConversionError, RateConversionError +from .exception import ConversionError, RateConversionError, TariffAccessDenied from .scenario import Scenario from .shared import as_naive_datetime @@ -37,7 +37,14 @@ def tariff_iter_rates_for_dt( if rate["rate_bands"]: yield rate elif rider_id := rate.get("rider_id"): - rider_tariff = library.tariffs.get_tariff(rider_id) + try: + rider_tariff = library.tariffs.get_tariff(rider_id) + except TariffAccessDenied: + library.record_issue( + ("inaccessible_rider", rate["tariff_rate_id"], rider_id), + f"Skipping inaccessible rider {rider_id} attached to rate {rate['tariff_rate_id']} ({rate['rate_name']})", + ) + continue yield from tariff_iter_rates_for_dt(rider_tariff, scenario, library, dt) @@ -104,8 +111,6 @@ def rate_filter_bands(rate: TariffRateExtended, _scenario: Scenario, library: Li raise RateConversionError(rate, "Bands with property limits are not supported") if band.get("prev_upper_limit"): raise RateConversionError(rate, "Bands with property prev_upper_limit are not supported") - if band.get("calculation_factor"): - raise RateConversionError(rate, "Bands with property calculation_factor are not supported") if band.get("applicability_formula"): raise RateConversionError(rate, "Bands with property applicability_formula are not supported") if (applicability_value := band.get("applicability_value")) is not None: @@ -173,8 +178,12 @@ def rate_band_get_amount_at_datetime(band: TariffRateBand, library: Library, dt: raise RateConversionError(rate, "Rates with variable_factor_key are not supported") mp = -1 if band["is_credit"] else 1 if (variable_rate_key := rate.get("variable_rate_key")) is None: - return band["rate_amount"] * mp - return library.variables.lookup(variable_rate_key, dt) * mp + raw_result = band["rate_amount"] * mp + else: + raw_result = library.variables.lookup(variable_rate_key, dt) * mp + + calculation_factor = band.get("calculation_factor", 1.0) + return raw_result * calculation_factor # ================================ diff --git a/tests/test_arcadia_urdb_fixedcharge.py b/tests/test_arcadia_urdb_fixedcharge.py index ba8d0a6..6a5ac1e 100644 --- a/tests/test_arcadia_urdb_fixedcharge.py +++ b/tests/test_arcadia_urdb_fixedcharge.py @@ -47,6 +47,28 @@ def test_get_rate_fixed_charge_converts_daily_to_monthly(monkeypatch): assert result == 56.0 +def test_get_rate_fixed_charge_applies_calculation_factor(monkeypatch): + rate = make_fixed_rate(rate_bands=[make_band(rate_amount=2.0, calculation_factor=1.5)]) + + monkeypatch.setattr( + "tariff_fetch.urdb.arcadia.fixedcharge.ru.rate_filter_bands", + lambda rate, scenario, library: list(rate["rate_bands"]), + ) + monkeypatch.setattr( + "tariff_fetch.urdb.arcadia.fixedcharge.ru.rate_band_get_amount_at_datetime", + lambda band, library, dt: band["rate_amount"] * band["calculation_factor"], + ) + + result = get_rate_fixed_charge_at_dt( + scenario=None, # pyright: ignore[reportArgumentType] + library=None, # pyright: ignore[reportArgumentType] + rate=rate, # pyright: ignore[reportArgumentType] + dt=datetime(2025, 2, 1, 0, 30), + ) + + assert result == 3.0 + + def test_normalize_fixed_charge_amount_rejects_unsupported_period(): with pytest.raises(ValueError, match="Unsupported fixed charge period"): normalize_fixed_charge_amount(1.0, "YEARLY", datetime(2025, 1, 1, 0, 30)) diff --git a/tests/test_arcadia_urdb_rateutils.py b/tests/test_arcadia_urdb_rateutils.py index 5dad26f..3a305eb 100644 --- a/tests/test_arcadia_urdb_rateutils.py +++ b/tests/test_arcadia_urdb_rateutils.py @@ -3,12 +3,14 @@ from types import SimpleNamespace import pytest - +from requests import HTTPError, Response from tariff_fetch.urdb.arcadia import energyschedule as es from tariff_fetch.urdb.arcadia import rateutils as ru +from tariff_fetch.urdb.arcadia.exception import TariffAccessDenied from tariff_fetch.urdb.arcadia.exception import RateConversionError +from tariff_fetch.urdb.arcadia.library import LibraryDebugStore, TariffLibrary from tariff_fetch.urdb.arcadia.scenario import Scenario -from tests.arcadia_urdb_fixtures import StubLibrary, make_band, make_consumption_rate, make_percentage_rate +from tests.arcadia_urdb_fixtures import StubLibrary, make_band, make_consumption_rate, make_percentage_rate, make_rate def test_rate_filter_bands_excludes_non_matching_choice_band(): @@ -176,6 +178,51 @@ def test_rate_band_get_amount_at_datetime_rejects_variable_rate_sub_key(): ) +def test_rate_band_get_amount_at_datetime_applies_calculation_factor_to_fixed_amount(): + band = make_band(tariff_rate_id=10, rate_amount=5.0, calculation_factor=1.5) + rate = make_consumption_rate(tariff_rate_id=10) + library = SimpleNamespace(tariffs=SimpleNamespace(get_rate=lambda rate_id: rate)) + + result = ru.rate_band_get_amount_at_datetime( + band, # type: ignore[arg-type] + library, # type: ignore[arg-type] + datetime(2025, 1, 1, 0, 30), + ) + + assert result == 7.5 + + +def test_rate_band_get_amount_at_datetime_applies_calculation_factor_to_credit_band(): + band = make_band(tariff_rate_id=11, rate_amount=5.0, calculation_factor=1.5, is_credit=True) + rate = make_consumption_rate(tariff_rate_id=11) + library = SimpleNamespace(tariffs=SimpleNamespace(get_rate=lambda rate_id: rate)) + + result = ru.rate_band_get_amount_at_datetime( + band, # type: ignore[arg-type] + library, # type: ignore[arg-type] + datetime(2025, 1, 1, 0, 30), + ) + + assert result == -7.5 + + +def test_rate_band_get_amount_at_datetime_applies_calculation_factor_to_variable_rate(): + band = make_band(tariff_rate_id=12, calculation_factor=2.0) + rate = make_consumption_rate(tariff_rate_id=12, variable_rate_key="monthlySupply") + library = SimpleNamespace( + tariffs=SimpleNamespace(get_rate=lambda rate_id: rate), + variables=SimpleNamespace(lookup=lambda key, dt: 3.5), + ) + + result = ru.rate_band_get_amount_at_datetime( + band, # type: ignore[arg-type] + library, # type: ignore[arg-type] + datetime(2025, 1, 1, 0, 30), + ) + + assert result == 7.0 + + def test_get_rate_consumption_bands_rejects_quantity_key(): rate = make_consumption_rate(quantity_key="billingMeter") @@ -186,3 +233,55 @@ def test_get_rate_consumption_bands_rejects_quantity_key(): library=None, # pyright: ignore[reportArgumentType] dt=datetime(2025, 1, 1, 0, 30), ) + + +def test_tariff_iter_rates_for_dt_skips_inaccessible_rider_and_records_issue(): + rate = make_rate(tariff_rate_id=9, rate_name="Rider Placeholder", rate_bands=[], rider_id=77) + tariff = {"rates": [rate]} + issues: list[tuple[tuple[object, ...], str]] = [] + + def get_tariff(tariff_id: int): + raise TariffAccessDenied(tariff_id) + + library = SimpleNamespace( + tariffs=SimpleNamespace(get_tariff=get_tariff), + record_issue=lambda key, message: issues.append((key, message)), + ) + + result = list( + ru.tariff_iter_rates_for_dt( + tariff, # type: ignore[arg-type] + Scenario(1, 2025, False, {"SUPPLY"}), + library, # type: ignore[arg-type] + datetime(2025, 1, 1, 0, 30), + ) + ) + + assert result == [] + assert issues == [ + ( + ("inaccessible_rider", 9, 77), + "Skipping inaccessible rider 77 attached to rate 9 (Rider Placeholder)", + ) + ] + + +def test_tariff_library_caches_access_denied_tariff_ids(): + calls: list[int] = [] + + class DummyTariffsAPI: + def iter_pages(self, **kwargs): + calls.append(int(kwargs["search"])) + response = Response() + response.status_code = 403 + raise HTTPError(response=response) + + api = SimpleNamespace(tariffs=DummyTariffsAPI()) + library = TariffLibrary(api, LibraryDebugStore()) # type: ignore[arg-type] + + with pytest.raises(TariffAccessDenied): + library.get_tariff(77) + with pytest.raises(TariffAccessDenied): + library.get_tariff(77) + + assert calls == [77] diff --git a/uv.lock b/uv.lock index 2fdb5dc..6cee81c 100644 --- a/uv.lock +++ b/uv.lock @@ -1480,7 +1480,7 @@ wheels = [ [[package]] name = "tariff-fetch" -version = "0.4.7" +version = "0.4.8" source = { editable = "." } dependencies = [ { name = "fastexcel" }, From 3adc5bc2ad78a63bd4bfc48bc8b0dd322a46f6a7 Mon Sep 17 00:00:00 2001 From: Natalia Borovskikh Date: Thu, 12 Mar 2026 15:23:09 +0000 Subject: [PATCH 21/24] Record calendar_id cases as an issue instead of hard failing --- docs/providers/arcadia/urdb-converter.md | 17 ++++- tariff_fetch/urdb/arcadia/rateutils.py | 27 ++++++-- tests/test_arcadia_urdb_boundaries_build.py | 73 ++++++++++++++++++--- 3 files changed, 103 insertions(+), 14 deletions(-) diff --git a/docs/providers/arcadia/urdb-converter.md b/docs/providers/arcadia/urdb-converter.md index 91e0086..1a5c5cd 100644 --- a/docs/providers/arcadia/urdb-converter.md +++ b/docs/providers/arcadia/urdb-converter.md @@ -69,7 +69,6 @@ Examples include: - rates whose tier limits come from a variable lookup - rates whose value must be multiplied by an additional variable factor - rates that depend on a quantity such as number of meters -- calendar-driven time-of-use overrides such as holiday or event calendars - demand-based rate bands - property-limited bands and formula-driven band logic @@ -115,6 +114,22 @@ This includes: This folder is useful when checking why a tariff converted a certain way or why the converter stopped on a particular feature. +## Warnings and skipped items + +Some Arcadia features are not converted exactly, but also do not stop the whole conversion. + +Current examples: + +- inaccessible rider tariffs that Arcadia returns as unauthorized +- time-of-use calendars attached through `calendar_id` + +When this happens, the converter records a warning for the conversion run. + +- inaccessible riders are skipped rather than retried forever +- TOU calendar ids are ignored, and the converter falls back to the ordinary TOU period definitions + +These warnings are meant to make the approximation visible without turning every such case into a hard failure. + ## Partial conversions The current interactive flow may allow you to continue after one part of the conversion fails. diff --git a/tariff_fetch/urdb/arcadia/rateutils.py b/tariff_fetch/urdb/arcadia/rateutils.py index 0198a34..a6ba8fd 100644 --- a/tariff_fetch/urdb/arcadia/rateutils.py +++ b/tariff_fetch/urdb/arcadia/rateutils.py @@ -11,7 +11,7 @@ from tariff_fetch.arcadia.schema.timeofuse import Period, TimeOfUseExtended from tariff_fetch.urdb.arcadia.library import Library -from .exception import ConversionError, RateConversionError, TariffAccessDenied +from .exception import RateConversionError, TariffAccessDenied from .scenario import Scenario from .shared import as_naive_datetime @@ -32,6 +32,7 @@ def tariff_iter_rates_for_dt( for rate in rates: if not rate_is_applied_to_scenario(rate, scenario, library): continue + _record_calendar_issues(rate, library) if not rate_is_applied_to_datetime(rate, dt): continue if rate["rate_bands"]: @@ -197,8 +198,6 @@ def time_of_use_is_datetime_within( ) -> bool: """Return whether a datetime falls within a supported Arcadia TOU definition.""" - if tou.get("calendar_id") is not None: - raise ConversionError("TOU calendar_id is not supported") season = tou.get("season") if season and not season_is_datetime_within(season, dt): return False @@ -217,8 +216,6 @@ def period_is_datetime_within( ) -> bool: """Return whether a datetime falls within a TOU period's weekday and time bounds.""" - if period.get("calendar_id") is not None: - raise ConversionError("TOU period calendar_id is not supported") return ( period["from_day_of_week"] <= dt.weekday() <= period["to_day_of_week"] and period["from_hour"] <= dt.hour <= period["to_hour"] @@ -226,6 +223,26 @@ def period_is_datetime_within( ) +def _record_calendar_issues(rate: TariffRateExtended, library: Library) -> None: + """Record ignored TOU calendar usage once per conversion run.""" + + if (tou := rate.get("time_of_use")) is None: + return + if (calendar_id := tou.get("calendar_id")) is not None: + library.record_issue( + ("ignored_tou_calendar", rate["tariff_rate_id"], calendar_id), + f"Ignoring TOU calendar_id {calendar_id} for rate {rate['tariff_rate_id']} ({rate['rate_name']})", + ) + for period in tou.get("tou_periods", []): + if (calendar_id := period.get("calendar_id")) is None: + continue + period_id = period.get("tou_period_id", "unknown") + library.record_issue( + ("ignored_tou_period_calendar", rate["tariff_rate_id"], period_id, calendar_id), + f"Ignoring TOU period calendar_id {calendar_id} for rate {rate['tariff_rate_id']} ({rate['rate_name']})", + ) + + # ================================ # Season # ================================ diff --git a/tests/test_arcadia_urdb_boundaries_build.py b/tests/test_arcadia_urdb_boundaries_build.py index 1421cea..78542d7 100644 --- a/tests/test_arcadia_urdb_boundaries_build.py +++ b/tests/test_arcadia_urdb_boundaries_build.py @@ -6,7 +6,6 @@ from tariff_fetch.urdb.arcadia import build as build_mod from tariff_fetch.urdb.arcadia import metadata as metadata_mod from tariff_fetch.urdb.arcadia import rateutils as ru -from tariff_fetch.urdb.arcadia.exception import ConversionError from tariff_fetch.urdb.arcadia.scenario import Scenario @@ -130,17 +129,25 @@ def test_rate_is_applied_to_datetime_handles_offset_aware_rate_window(): assert ru.rate_is_applied_to_datetime(rate, datetime(2025, 6, 1, 0, 0)) is False # type: ignore[arg-type] -def test_time_of_use_is_datetime_within_rejects_calendar_id(): +def test_time_of_use_is_datetime_within_ignores_calendar_id(): tou = { "calendar_id": 10, - "tou_periods": [], + "tou_periods": [ + { + "from_day_of_week": 0, + "to_day_of_week": 6, + "from_hour": 0, + "to_hour": 23, + "from_minute": 0, + "to_minute": 59, + } + ], } - with pytest.raises(ConversionError, match="calendar_id"): - ru.time_of_use_is_datetime_within(tou, datetime(2025, 7, 10, 15, 30)) # type: ignore[arg-type] + assert ru.time_of_use_is_datetime_within(tou, datetime(2025, 7, 10, 15, 30)) is True # type: ignore[arg-type] -def test_period_is_datetime_within_rejects_calendar_id(): +def test_period_is_datetime_within_ignores_calendar_id(): period = { "from_day_of_week": 0, "to_day_of_week": 6, @@ -151,8 +158,58 @@ def test_period_is_datetime_within_rejects_calendar_id(): "calendar_id": 10, } - with pytest.raises(ConversionError, match="calendar_id"): - ru.period_is_datetime_within(period, datetime(2025, 7, 10, 15, 30)) # type: ignore[arg-type] + assert ru.period_is_datetime_within(period, datetime(2025, 7, 10, 15, 30)) is True # type: ignore[arg-type] + + +def test_tariff_iter_rates_for_dt_records_ignored_calendar_issues_once(): + rate = { + "tariff_rate_id": 10, + "rate_name": "TOU Charge", + "charge_class": ["SUPPLY"], + "rate_bands": [{"rate_unit": "COST_PER_UNIT"}], + "time_of_use": { + "calendar_id": 10, + "tou_periods": [ + { + "tou_period_id": 33, + "from_day_of_week": 0, + "to_day_of_week": 6, + "from_hour": 0, + "to_hour": 23, + "from_minute": 0, + "to_minute": 59, + "calendar_id": 11, + } + ], + }, + } + messages: list[tuple[tuple[object, ...], str]] = [] + library = SimpleNamespace( + record_issue=lambda key, message: messages.append((key, message)), + get_choice_property_as_ints=lambda key: [1], + ) + tariff = {"rates": [rate]} + scenario = Scenario(1, 2025, False, {"SUPPLY"}) + + _ = list( + ru.tariff_iter_rates_for_dt( + tariff, # type: ignore[arg-type] + scenario, + library, # type: ignore[arg-type] + datetime(2025, 7, 10, 15, 30), + ) + ) + + assert messages == [ + ( + ("ignored_tou_calendar", 10, 10), + "Ignoring TOU calendar_id 10 for rate 10 (TOU Charge)", + ), + ( + ("ignored_tou_period_calendar", 10, 33, 11), + "Ignoring TOU period calendar_id 11 for rate 10 (TOU Charge)", + ), + ] def test_build_urdb_merges_converter_chunks(monkeypatch): From f309af904a4675e0d5953916a8e7a896deb696a1 Mon Sep 17 00:00:00 2001 From: Natalia Borovskikh Date: Fri, 13 Mar 2026 13:40:55 +0000 Subject: [PATCH 22/24] Fix time-of-use period boundaries --- tariff_fetch/urdb/arcadia/rateutils.py | 23 +++- tests/test_arcadia_urdb_boundaries_build.py | 110 ++++++++++++++++++-- 2 files changed, 120 insertions(+), 13 deletions(-) diff --git a/tariff_fetch/urdb/arcadia/rateutils.py b/tariff_fetch/urdb/arcadia/rateutils.py index a6ba8fd..836f5d0 100644 --- a/tariff_fetch/urdb/arcadia/rateutils.py +++ b/tariff_fetch/urdb/arcadia/rateutils.py @@ -216,11 +216,24 @@ def period_is_datetime_within( ) -> bool: """Return whether a datetime falls within a TOU period's weekday and time bounds.""" - return ( - period["from_day_of_week"] <= dt.weekday() <= period["to_day_of_week"] - and period["from_hour"] <= dt.hour <= period["to_hour"] - and period["from_minute"] <= dt.minute <= period["to_minute"] - ) + current_day = dt.weekday() + start_day, end_day = period["from_day_of_week"], period["to_day_of_week"] + + if start_day <= end_day: + if not (start_day <= current_day <= end_day): + return False + elif not (start_day <= current_day or current_day <= end_day): + return False + + current_time = dt.hour * 60 + dt.minute + start_time = period["from_hour"] * 60 + period["from_minute"] + end_time = period["to_hour"] * 60 + period["to_minute"] + + if start_time == end_time: + return True + if start_time < end_time: + return start_time <= current_time < end_time + return current_time >= start_time or current_time < end_time def _record_calendar_issues(rate: TariffRateExtended, library: Library) -> None: diff --git a/tests/test_arcadia_urdb_boundaries_build.py b/tests/test_arcadia_urdb_boundaries_build.py index 78542d7..dd41e69 100644 --- a/tests/test_arcadia_urdb_boundaries_build.py +++ b/tests/test_arcadia_urdb_boundaries_build.py @@ -35,20 +35,114 @@ def test_season_is_datetime_within_handles_wraparound_year(): assert ru.season_is_datetime_within(season, date(2025, 5, 1)) is False # type: ignore[arg-type] -def test_period_is_datetime_within_respects_inclusive_boundaries(): +def test_period_is_datetime_within_respects_exclusive_to_portion(): period = { "from_day_of_week": 0, - "to_day_of_week": 4, - "from_hour": 9, - "to_hour": 17, + "to_day_of_week": 0, + "from_hour": 0, + "to_hour": 0, + "from_minute": 0, + "to_minute": 0, + } + # Monday 00:00 + assert ru.period_is_datetime_within(period, datetime(2025, 6, 2, 0, 0)) is True + # Monday 12:00 + assert ru.period_is_datetime_within(period, datetime(2025, 6, 2, 12, 0)) is True + # Monday 23:59 + assert ru.period_is_datetime_within(period, datetime(2025, 6, 2, 23, 59)) is True + # Tuesday 00:00 + assert ru.period_is_datetime_within(period, datetime(2025, 6, 3, 0, 0)) is False + # Tuesday 06:00 + assert ru.period_is_datetime_within(period, datetime(2025, 6, 3, 12, 0)) is False + + +def test_period_is_datetime_within_excludes_exact_to_boundary_for_non_wrapping_window(): + period = { + "from_day_of_week": 0, + "to_day_of_week": 0, + "from_hour": 14, + "to_hour": 18, "from_minute": 0, "to_minute": 59, } - assert ru.period_is_datetime_within(period, datetime(2025, 1, 6, 9, 0)) is True # type: ignore[arg-type] - assert ru.period_is_datetime_within(period, datetime(2025, 1, 10, 17, 59)) is True # type: ignore[arg-type] - assert ru.period_is_datetime_within(period, datetime(2025, 1, 11, 12, 0)) is False # type: ignore[arg-type] - assert ru.period_is_datetime_within(period, datetime(2025, 1, 6, 8, 59)) is False # type: ignore[arg-type] + assert ru.period_is_datetime_within(period, datetime(2025, 6, 2, 18, 58)) is True + assert ru.period_is_datetime_within(period, datetime(2025, 6, 2, 18, 59)) is False + + +def test_period_is_datetime_within_respects_day_isolation(): + period = { + "from_day_of_week": 0, + "to_day_of_week": 2, + "from_hour": 17, + "to_hour": 8, + "from_minute": 0, + "to_minute": 0, + } + # Monday, 00:00 + assert ru.period_is_datetime_within(period, datetime(2025, 6, 2, 0, 0)) is True + # Monday, 06:00 + assert ru.period_is_datetime_within(period, datetime(2025, 6, 2, 6, 0)) is True + # Monday, 09:00 + assert ru.period_is_datetime_within(period, datetime(2025, 6, 2, 9, 0)) is False + # Monday, 16:00 + assert ru.period_is_datetime_within(period, datetime(2025, 6, 2, 16, 0)) is False + # Monday, 17:00 + assert ru.period_is_datetime_within(period, datetime(2025, 6, 2, 17, 0)) is True + # Monday, 23:00 + assert ru.period_is_datetime_within(period, datetime(2025, 6, 2, 23, 0)) is True + # Tuesday, 00:00 + assert ru.period_is_datetime_within(period, datetime(2025, 6, 3, 0, 0)) is True + # Tuesday, 06:00 + assert ru.period_is_datetime_within(period, datetime(2025, 6, 3, 6, 0)) is True + # Tuesday, 09:00 + assert ru.period_is_datetime_within(period, datetime(2025, 6, 3, 9, 0)) is False + # Tuesday, 16:00 + assert ru.period_is_datetime_within(period, datetime(2025, 6, 3, 16, 0)) is False + # Tuesday, 17:00 + assert ru.period_is_datetime_within(period, datetime(2025, 6, 3, 17, 0)) is True + # Tuesday, 23:00 + assert ru.period_is_datetime_within(period, datetime(2025, 6, 3, 23, 0)) is True + # Wednesday, 00:00 + assert ru.period_is_datetime_within(period, datetime(2025, 6, 4, 0, 0)) is True + # Wednesday, 06:00 + assert ru.period_is_datetime_within(period, datetime(2025, 6, 4, 6, 0)) is True + # Wednesday, 09:00 + assert ru.period_is_datetime_within(period, datetime(2025, 6, 4, 9, 0)) is False + # Wednesday, 16:00 + assert ru.period_is_datetime_within(period, datetime(2025, 6, 4, 16, 0)) is False + # Wednesday, 17:00 + assert ru.period_is_datetime_within(period, datetime(2025, 6, 4, 17, 0)) is True + # Wednesday, 23:00 + assert ru.period_is_datetime_within(period, datetime(2025, 6, 4, 23, 0)) is True + # Thursday, 00:00 + assert ru.period_is_datetime_within(period, datetime(2025, 6, 5, 0, 0)) is False + # Thursday, 06:00 + assert ru.period_is_datetime_within(period, datetime(2025, 6, 5, 6, 0)) is False + # Thursday, 09:00 + assert ru.period_is_datetime_within(period, datetime(2025, 6, 5, 9, 0)) is False + # Thursday, 16:00 + assert ru.period_is_datetime_within(period, datetime(2025, 6, 5, 16, 0)) is False + # Thursday, 17:00 + assert ru.period_is_datetime_within(period, datetime(2025, 6, 5, 17, 0)) is False + # Thursday, 23:00 + assert ru.period_is_datetime_within(period, datetime(2025, 6, 5, 23, 0)) is False + + +def test_period_is_datetime_within_excludes_exact_to_boundary_for_wrapping_window(): + period = { + "from_day_of_week": 0, + "to_day_of_week": 2, + "from_hour": 17, + "to_hour": 8, + "from_minute": 0, + "to_minute": 0, + } + + assert ru.period_is_datetime_within(period, datetime(2025, 6, 2, 7, 59)) is True + assert ru.period_is_datetime_within(period, datetime(2025, 6, 2, 8, 0)) is False + assert ru.period_is_datetime_within(period, datetime(2025, 6, 4, 7, 59)) is True + assert ru.period_is_datetime_within(period, datetime(2025, 6, 4, 8, 0)) is False def test_time_of_use_is_datetime_within_applies_embedded_season(): From 508adcd27fdfbdf4f54429b529d7fa60df4dbf22 Mon Sep 17 00:00:00 2001 From: Natalia Borovskikh Date: Mon, 16 Mar 2026 12:04:36 +0000 Subject: [PATCH 23/24] Bump version --- pyproject.toml | 2 +- uv.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d2e6e7d..0bfc737 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "tariff_fetch" -version = "0.4.8" +version = "0.4.9" description = "A CLI tool and a python library to simplify downloading electric and gas utility tariff data." authors = [{ name = "Switchbox", email = "hello@switch.box" }] readme = "README.md" diff --git a/uv.lock b/uv.lock index 6cee81c..46d7bec 100644 --- a/uv.lock +++ b/uv.lock @@ -1480,7 +1480,7 @@ wheels = [ [[package]] name = "tariff-fetch" -version = "0.4.8" +version = "0.4.9" source = { editable = "." } dependencies = [ { name = "fastexcel" }, From 62720a3ae45298379a09da3a1143f477fec6bd8f Mon Sep 17 00:00:00 2001 From: Natalia Borovskikh Date: Mon, 16 Mar 2026 12:09:34 +0000 Subject: [PATCH 24/24] Fix formatting issues --- pyproject.toml | 2 +- tariff_fetch/_cli/openei.py | 4 ++-- tariff_fetch/arcadia/api.py | 3 ++- tariff_fetch/arcadia/schema/lookup.py | 3 ++- tariff_fetch/arcadia/schema/lse.py | 3 ++- tariff_fetch/arcadia/schema/season.py | 3 ++- tariff_fetch/arcadia/schema/tariff.py | 3 ++- tariff_fetch/arcadia/schema/tariffproperty.py | 3 ++- tariff_fetch/arcadia/schema/tariffrate.py | 3 ++- tariff_fetch/arcadia/schema/territory.py | 3 ++- tariff_fetch/arcadia/schema/timeofuse.py | 3 ++- tariff_fetch/openei/utility_rates.py | 6 +++--- tariff_fetch/rateacuity/report_tables.py | 3 ++- tariff_fetch/rateacuity/schema.py | 3 ++- tariff_fetch/urdb/arcadia/library.py | 7 +++++-- tariff_fetch/urdb/rateacuity_history_gas/exceptions.py | 4 +++- tariff_fetch/urdb/rateacuity_history_gas/types.py | 4 ++-- tariff_fetch/urdb/schema.py | 3 ++- tests/test_arcadia_urdb_boundaries_build.py | 8 +++----- tests/test_arcadia_urdb_rateutils.py | 4 ++-- 20 files changed, 45 insertions(+), 30 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0bfc737..e2a3666 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,7 @@ testpaths = ["tests"] addopts = ["-q"] [tool.ruff] -target-version = "py310" +target-version = "py311" line-length = 120 fix = true diff --git a/tariff_fetch/_cli/openei.py b/tariff_fetch/_cli/openei.py index efd7448..99f326c 100644 --- a/tariff_fetch/_cli/openei.py +++ b/tariff_fetch/_cli/openei.py @@ -1,6 +1,6 @@ import json import os -from datetime import datetime, timezone +from datetime import UTC, datetime from pathlib import Path from typing import Literal, cast @@ -47,7 +47,7 @@ def _get_tariffs( with console.status("Fetching rates..."): iterator = iter_utility_rates( api_key, - effective_on_date=datetime.now(timezone.utc), + effective_on_date=datetime.now(UTC), sector=sector, detail=detail, eia=eia_id, diff --git a/tariff_fetch/arcadia/api.py b/tariff_fetch/arcadia/api.py index c86892b..a500374 100644 --- a/tariff_fetch/arcadia/api.py +++ b/tariff_fetch/arcadia/api.py @@ -2,13 +2,14 @@ from collections.abc import Iterator from dataclasses import dataclass, field from datetime import date -from typing import Annotated, Any, Generic, Literal, Self, TypedDict, TypeVar, Unpack, overload +from typing import Annotated, Any, Generic, Literal, Self, TypeVar, Unpack, overload from urllib.parse import urljoin import requests from pydantic import ConfigDict, PlainSerializer, TypeAdapter from pydantic.alias_generators import to_camel from requests.auth import HTTPBasicAuth +from typing_extensions import TypedDict from .schema.common import CustomerClass, TariffType from .schema.lookup import Lookup diff --git a/tariff_fetch/arcadia/schema/lookup.py b/tariff_fetch/arcadia/schema/lookup.py index 534e333..fedd948 100644 --- a/tariff_fetch/arcadia/schema/lookup.py +++ b/tariff_fetch/arcadia/schema/lookup.py @@ -1,7 +1,8 @@ from datetime import datetime -from typing import NotRequired, TypedDict +from typing import NotRequired from pydantic.alias_generators import to_camel +from typing_extensions import TypedDict class Lookup(TypedDict): diff --git a/tariff_fetch/arcadia/schema/lse.py b/tariff_fetch/arcadia/schema/lse.py index 04ece35..dcbd39c 100644 --- a/tariff_fetch/arcadia/schema/lse.py +++ b/tariff_fetch/arcadia/schema/lse.py @@ -1,7 +1,8 @@ -from typing import Annotated, Literal, TypedDict +from typing import Annotated, Literal from pydantic import BeforeValidator from pydantic.alias_generators import to_camel +from typing_extensions import TypedDict from .validators import comma_separated_str diff --git a/tariff_fetch/arcadia/schema/season.py b/tariff_fetch/arcadia/schema/season.py index 05cf6f7..b414be5 100644 --- a/tariff_fetch/arcadia/schema/season.py +++ b/tariff_fetch/arcadia/schema/season.py @@ -1,7 +1,8 @@ -from typing import Annotated, NotRequired, TypedDict +from typing import Annotated, NotRequired from pydantic import Field from pydantic.alias_generators import to_camel +from typing_extensions import TypedDict from .common import SeasonPredominance diff --git a/tariff_fetch/arcadia/schema/tariff.py b/tariff_fetch/arcadia/schema/tariff.py index 8e57901..2f0dc7b 100644 --- a/tariff_fetch/arcadia/schema/tariff.py +++ b/tariff_fetch/arcadia/schema/tariff.py @@ -1,8 +1,9 @@ from datetime import date, datetime -from typing import Annotated, NotRequired, TypedDict +from typing import Annotated, NotRequired from pydantic import BeforeValidator, FailFast from pydantic.alias_generators import to_camel +from typing_extensions import TypedDict from .common import ( CustomerClass, diff --git a/tariff_fetch/arcadia/schema/tariffproperty.py b/tariff_fetch/arcadia/schema/tariffproperty.py index 9ba22b2..996b7a8 100644 --- a/tariff_fetch/arcadia/schema/tariffproperty.py +++ b/tariff_fetch/arcadia/schema/tariffproperty.py @@ -1,6 +1,7 @@ -from typing import Literal, NotRequired, TypedDict +from typing import Literal, NotRequired from pydantic.alias_generators import to_camel +from typing_extensions import TypedDict TariffPropertyFormulaType = Literal["FORMULA"] diff --git a/tariff_fetch/arcadia/schema/tariffrate.py b/tariff_fetch/arcadia/schema/tariffrate.py index f54cb3f..0fea06d 100644 --- a/tariff_fetch/arcadia/schema/tariffrate.py +++ b/tariff_fetch/arcadia/schema/tariffrate.py @@ -1,8 +1,9 @@ from datetime import datetime -from typing import Annotated, NotRequired, TypedDict +from typing import Annotated, NotRequired from pydantic import BeforeValidator from pydantic.alias_generators import to_camel +from typing_extensions import TypedDict from .common import RateChargeClass, RateTransactionType, RateUnit, TariffChargePeriod, TariffChargeType from .season import SeasonExtended, SeasonStandard diff --git a/tariff_fetch/arcadia/schema/territory.py b/tariff_fetch/arcadia/schema/territory.py index 7300b95..89bde30 100644 --- a/tariff_fetch/arcadia/schema/territory.py +++ b/tariff_fetch/arcadia/schema/territory.py @@ -1,6 +1,7 @@ -from typing import NotRequired, TypedDict +from typing import NotRequired from pydantic.alias_generators import to_camel +from typing_extensions import TypedDict from .common import TerritoryItemType, TerritoryUsageType diff --git a/tariff_fetch/arcadia/schema/timeofuse.py b/tariff_fetch/arcadia/schema/timeofuse.py index 0742bd3..689ab6a 100644 --- a/tariff_fetch/arcadia/schema/timeofuse.py +++ b/tariff_fetch/arcadia/schema/timeofuse.py @@ -1,7 +1,8 @@ -from typing import Annotated, TypedDict +from typing import Annotated from pydantic import Field from pydantic.alias_generators import to_camel +from typing_extensions import TypedDict from .common import TimeOfUsePrivacy, TimeOfUseType from .season import SeasonExtended, SeasonStandard diff --git a/tariff_fetch/openei/utility_rates.py b/tariff_fetch/openei/utility_rates.py index 4842141..6c3cac7 100644 --- a/tariff_fetch/openei/utility_rates.py +++ b/tariff_fetch/openei/utility_rates.py @@ -4,9 +4,9 @@ import os from collections.abc import Iterator, Sequence from pathlib import Path -from typing import Any, Literal, TypeAlias, TypedDict, cast +from typing import Any, Literal, TypeAlias, cast -from typing_extensions import Unpack +from typing_extensions import TypedDict, Unpack from .base import api_request_json # pyright: ignore[reportUnknownVariableType] @@ -330,7 +330,7 @@ def main(argv: Sequence[str] | None = None) -> None: if rates_for_utility is None: parser.error("ratesforutility must be set") - effective_on_date = int(datetime.datetime.now(datetime.timezone.utc).timestamp()) + effective_on_date = int(datetime.datetime.now(datetime.UTC).timestamp()) records = list( iter_utility_rates( api_key=api_key, diff --git a/tariff_fetch/rateacuity/report_tables.py b/tariff_fetch/rateacuity/report_tables.py index 26d3bfb..23a97af 100644 --- a/tariff_fetch/rateacuity/report_tables.py +++ b/tariff_fetch/rateacuity/report_tables.py @@ -1,10 +1,11 @@ from __future__ import annotations -from typing import TypedDict, cast +from typing import cast from selenium.webdriver.common.by import By from selenium.webdriver.remote.webdriver import WebDriver from selenium.webdriver.remote.webelement import WebElement +from typing_extensions import TypedDict from .schema import Section diff --git a/tariff_fetch/rateacuity/schema.py b/tariff_fetch/rateacuity/schema.py index a4728b9..8178da7 100644 --- a/tariff_fetch/rateacuity/schema.py +++ b/tariff_fetch/rateacuity/schema.py @@ -1,9 +1,10 @@ import re from datetime import date -from typing import Annotated, Any, Generic, Literal, NamedTuple, TypedDict, TypeVar +from typing import Annotated, Any, Generic, Literal, NamedTuple, TypeVar from pydantic import BeforeValidator, Field from pydantic.alias_generators import to_camel +from typing_extensions import TypedDict USStateCode = Literal[ "AL", diff --git a/tariff_fetch/urdb/arcadia/library.py b/tariff_fetch/urdb/arcadia/library.py index 949f2f7..24db4c4 100644 --- a/tariff_fetch/urdb/arcadia/library.py +++ b/tariff_fetch/urdb/arcadia/library.py @@ -6,6 +6,7 @@ from typing import Literal, final, overload from requests import HTTPError + from tariff_fetch.arcadia.api import ArcadiaSignalAPI from tariff_fetch.arcadia.schema.lookup import Lookup from tariff_fetch.arcadia.schema.tariff import TariffExtended @@ -199,8 +200,10 @@ def lookup(self, key: str, dt: datetime) -> float: self.debug_store.save_lookups(key, dt.year, lookups) for row in lookups: - if as_naive_datetime(row["from_date_time"]) <= as_naive_datetime(dt) <= as_naive_datetime( - row["to_date_time"] or datetime.max + if ( + as_naive_datetime(row["from_date_time"]) + <= as_naive_datetime(dt) + <= as_naive_datetime(row["to_date_time"] or datetime.max) ): if (value := row["actual_value"]) is not None: return value diff --git a/tariff_fetch/urdb/rateacuity_history_gas/exceptions.py b/tariff_fetch/urdb/rateacuity_history_gas/exceptions.py index 76018ed..15e187d 100644 --- a/tariff_fetch/urdb/rateacuity_history_gas/exceptions.py +++ b/tariff_fetch/urdb/rateacuity_history_gas/exceptions.py @@ -1,4 +1,6 @@ -from typing_extensions import final, override +from typing import final + +from typing_extensions import override @final diff --git a/tariff_fetch/urdb/rateacuity_history_gas/types.py b/tariff_fetch/urdb/rateacuity_history_gas/types.py index 08714a1..770538a 100644 --- a/tariff_fetch/urdb/rateacuity_history_gas/types.py +++ b/tariff_fetch/urdb/rateacuity_history_gas/types.py @@ -1,7 +1,7 @@ from datetime import date -from typing import Literal, Required, TypedDict, get_args +from typing import Literal, Required, get_args -from typing_extensions import TypeIs +from typing_extensions import TypedDict, TypeIs BandDeterminant = Literal["ccf", "therms", "cubic feet"] ConsumptionRateDeterminant = Literal["per ccf", "per therm"] diff --git a/tariff_fetch/urdb/schema.py b/tariff_fetch/urdb/schema.py index 1063943..2c145e2 100644 --- a/tariff_fetch/urdb/schema.py +++ b/tariff_fetch/urdb/schema.py @@ -1,8 +1,9 @@ from __future__ import annotations -from typing import Annotated, Any, Literal, Required, TypedDict +from typing import Annotated, Any, Literal, Required from pydantic import FailFast, TypeAdapter +from typing_extensions import TypedDict # --- enums per URDB v7 docs --- RateSector = Literal["Residential", "Commercial", "Industrial", "Lighting"] diff --git a/tests/test_arcadia_urdb_boundaries_build.py b/tests/test_arcadia_urdb_boundaries_build.py index dd41e69..dd2bc3f 100644 --- a/tests/test_arcadia_urdb_boundaries_build.py +++ b/tests/test_arcadia_urdb_boundaries_build.py @@ -1,8 +1,6 @@ -from datetime import date, datetime, timezone +from datetime import UTC, date, datetime from types import SimpleNamespace -import pytest - from tariff_fetch.urdb.arcadia import build as build_mod from tariff_fetch.urdb.arcadia import metadata as metadata_mod from tariff_fetch.urdb.arcadia import rateutils as ru @@ -214,8 +212,8 @@ def test_rate_is_applied_to_datetime_combines_effective_window_with_season_and_t def test_rate_is_applied_to_datetime_handles_offset_aware_rate_window(): rate = { - "from_date_time": datetime(2025, 3, 1, 0, 0, tzinfo=timezone.utc), - "to_date_time": datetime(2025, 6, 1, 0, 0, tzinfo=timezone.utc), + "from_date_time": datetime(2025, 3, 1, 0, 0, tzinfo=UTC), + "to_date_time": datetime(2025, 6, 1, 0, 0, tzinfo=UTC), } assert ru.rate_is_applied_to_datetime(rate, datetime(2025, 2, 28, 23, 59)) is False # type: ignore[arg-type] diff --git a/tests/test_arcadia_urdb_rateutils.py b/tests/test_arcadia_urdb_rateutils.py index 3a305eb..803a7fc 100644 --- a/tests/test_arcadia_urdb_rateutils.py +++ b/tests/test_arcadia_urdb_rateutils.py @@ -4,10 +4,10 @@ import pytest from requests import HTTPError, Response + from tariff_fetch.urdb.arcadia import energyschedule as es from tariff_fetch.urdb.arcadia import rateutils as ru -from tariff_fetch.urdb.arcadia.exception import TariffAccessDenied -from tariff_fetch.urdb.arcadia.exception import RateConversionError +from tariff_fetch.urdb.arcadia.exception import RateConversionError, TariffAccessDenied from tariff_fetch.urdb.arcadia.library import LibraryDebugStore, TariffLibrary from tariff_fetch.urdb.arcadia.scenario import Scenario from tests.arcadia_urdb_fixtures import StubLibrary, make_band, make_consumption_rate, make_percentage_rate, make_rate