diff --git a/definitions.py b/definitions.py deleted file mode 100644 index 908b109a..00000000 --- a/definitions.py +++ /dev/null @@ -1,22 +0,0 @@ -""" -top-level folder to provide project-wide definitions & references -""" - -import os -from pathlib import Path - -PROJECT_ROOT = os.path.dirname(os.path.abspath(__file__)) - -OUTPUT_PATH: Path | None = None - - -def get_OUTPUT_PATH() -> Path: - if OUTPUT_PATH is None: - raise RuntimeError('Output path not yet defined') - # Use cast to help mypy understand the type - return OUTPUT_PATH - - -def set_OUTPUT_PATH(path: Path) -> None: - global OUTPUT_PATH - OUTPUT_PATH = path diff --git a/temoa/_internal/run_actions.py b/temoa/_internal/run_actions.py index 9815495e..a11b3ff3 100644 --- a/temoa/_internal/run_actions.py +++ b/temoa/_internal/run_actions.py @@ -1,32 +1,7 @@ """ Basic-level atomic functions that can be used by a sequencer, as needed - -Written by: J. F. Hyink -jeff@westernspark.us -https://westernspark.us -Created on: 11/15/23 - -Tools for Energy Model Optimization and Analysis (Temoa): -An open source framework for energy systems optimization modeling - -Copyright (C) 2015, NC State University - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -A complete copy of the GNU General Public License v2 (GPLv2) is available -in LICENSE.txt. Users uncompressing this from an archive may not have -received this license file. If not, see . """ -import os import sqlite3 import sys from logging import getLogger @@ -46,7 +21,6 @@ ) from pyomo.opt import SolverResults -import definitions from temoa._internal.table_writer import TableWriter from temoa.core.config import TemoaConfig from temoa.core.model import TemoaModel @@ -157,21 +131,9 @@ def build_instance( if not silent: try: # Check for warnings in log file to notify user. Ugly but it works - log_file = os.path.join(definitions.get_OUTPUT_PATH(), 'log.log') - with open(log_file) as f: - warnings_found = any( - '| WARNING |' in line or '| ERROR |' in line or '| CRITICAL |' in line - for line in f - ) - if warnings_found: - sys.stderr.write( - '\r[%8.2f] Instance created with warnings. Check log file.\n' - % (time() - hack) - ) - else: - sys.stderr.write( - '\r[%8.2f] Instance created. \n' % (time() - hack) - ) # needs spaces to clear previous line + sys.stderr.write( + '\r[%8.2f] Instance created. \n' % (time() - hack) + ) # needs spaces to clear previous line sys.stderr.flush() except Exception: sys.stderr.write( diff --git a/temoa/_internal/table_writer.py b/temoa/_internal/table_writer.py index 48ddc55c..d58d0ad3 100644 --- a/temoa/_internal/table_writer.py +++ b/temoa/_internal/table_writer.py @@ -2,10 +2,13 @@ tool for writing outputs to database tables """ +from __future__ import annotations + import sqlite3 import sys from collections import defaultdict from collections.abc import Iterable +from importlib import resources from logging import getLogger from pathlib import Path from typing import TYPE_CHECKING, cast @@ -13,7 +16,6 @@ from pyomo.core import value from pyomo.opt import SolverResults -from definitions import PROJECT_ROOT from temoa._internal.data_brick import DataBrick from temoa._internal.exchange_tech_cost_ledger import CostType from temoa._internal.table_data_puller import ( @@ -39,30 +41,6 @@ pass """ -Tools for Energy Model Optimization and Analysis (Temoa): -An open source framework for energy systems optimization modeling - -Copyright (C) 2015, NC State University - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -A complete copy of the GNU General Public License v2 (GPLv2) is available -in LICENSE.txt. Users uncompressing this from an archive may not have -received this license file. If not, see . - - -Written by: J. F. Hyink -jeff@westernspark.us -https://westernspark.us -Created on: 2/9/24 Note: This file borrows heavily from the legacy pformat_results.py, and is somewhat of a restructure of that code to accommodate the run modes more cleanly @@ -85,10 +63,11 @@ ] optional_output_tables = ['output_flow_out_summary', 'output_mc_delta', 'output_storage_level'] -flow_summary_file_loc = Path( - PROJECT_ROOT, 'temoa/extensions/modeling_to_generate_alternatives/make_flow_summary_table.sql' +flow_summary_file_loc = ( + resources.files('temoa.extensions.modeling_to_generate_alternatives') + / 'make_flow_summary_table.sql' ) -mc_tweaks_file_loc = Path(PROJECT_ROOT, 'temoa/extensions/monte_carlo/make_deltas_table.sql') +mc_tweaks_file_loc = resources.files('temoa.extensions.monte_carlo') / 'make_deltas_table.sql' class TableWriter: @@ -652,13 +631,16 @@ def make_mc_tweaks_table(self) -> None: # make the table for monte carlo tweaks, if needed... self.execute_script(mc_tweaks_file_loc) - def execute_script(self, script_file: str | Path) -> None: + def execute_script(self, script_file: str | Path | resources.abc.Traversable) -> None: """ A utility to execute a sql script on the current db connection :return: """ - with open(script_file) as table_script: - sql_commands = table_script.read() + if isinstance(script_file, resources.abc.Traversable): + sql_commands = script_file.read_text() + else: + with open(script_file) as table_script: + sql_commands = table_script.read() logger.debug('Executing sql from file: %s ', script_file) self.con.executescript(sql_commands) diff --git a/temoa/cli.py b/temoa/cli.py index a69456e0..79036b8f 100644 --- a/temoa/cli.py +++ b/temoa/cli.py @@ -8,7 +8,6 @@ from rich.logging import RichHandler from rich.text import Text -from definitions import set_OUTPUT_PATH from temoa._internal.temoa_sequencer import TemoaSequencer from temoa.core.config import TemoaConfig from temoa.core.modes import TemoaMode @@ -87,7 +86,6 @@ def _setup_sequencer( # Pass the silent flag to the logging setup _setup_logging(final_output_path, debug=debug, silent=silent) - set_OUTPUT_PATH(final_output_path) config = TemoaConfig.build_config( config_file=config_file, output_path=final_output_path, silent=silent ) diff --git a/temoa/extensions/method_of_morris/morris.py b/temoa/extensions/method_of_morris/morris.py index 0351a318..e8116ca3 100644 --- a/temoa/extensions/method_of_morris/morris.py +++ b/temoa/extensions/method_of_morris/morris.py @@ -1,10 +1,10 @@ # from __future__ import division import time +from importlib import resources from pathlib import Path from pyomo.dataportal import DataPortal -from definitions import PROJECT_ROOT from temoa._internal import run_actions from temoa._internal.table_writer import TableWriter from temoa.core.config import TemoaConfig @@ -63,12 +63,15 @@ def evaluate(param_names, param_values, data: dict, k): return Morris_Objectives -morris_root = Path(PROJECT_ROOT, 'temoa/extensions/method_of_morris') +morris_root = Path(__file__).parent perturbation_coefficient = 0.2 # minus plus 10% of the baseline values param_file = morris_root / 'm_params.txt' -db_file = Path(PROJECT_ROOT, 'data_files/untracked_data/morris/morris_utopia.sqlite') -config_path = Path(PROJECT_ROOT, 'data_files/untracked_data/morris/morris_utopia.toml') -with sqlite3.connect(db_file) as con: + +db_resource = resources.files('data_files.untracked_data.morris') / 'morris_utopia.sqlite' +config_resource = resources.files('data_files.untracked_data.morris') / 'morris_utopia.toml' +with resources.as_file(db_resource) as db_file, \ + resources.as_file(config_resource) as config_path, \ + sqlite3.connect(str(db_file)) as con: with open(param_file, 'w') as file: param_names = {} cur = con.cursor() diff --git a/temoa/extensions/method_of_morris/morris_sequencer.py b/temoa/extensions/method_of_morris/morris_sequencer.py index 9f41f98f..c788028c 100644 --- a/temoa/extensions/method_of_morris/morris_sequencer.py +++ b/temoa/extensions/method_of_morris/morris_sequencer.py @@ -1,35 +1,6 @@ """ -Tools for Energy Model Optimization and Analysis (Temoa): -An open source framework for energy systems optimization modeling - -Copyright (C) 2015, NC State University - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -A complete copy of the GNU General Public License v2 (GPLv2) is available -in LICENSE.txt. Users uncompressing this from an archive may not have -received this license file. If not, see . - -This code is modified from original work on Method of Morris framework -by Hadi Eshragi. Original morris.py file can be located in the energysystem -branch - -Modified/Refactored by: J. F. Hyink -jeff@westernspark.us -https://westernspark.us -Created on: 5/30/24 - An event sequencer to control the flow of a Method of Morris calculation. This code uses multiprocessing via the joblib library - """ import csv @@ -39,6 +10,7 @@ import sys import tomllib from logging.handlers import QueueListener +from importlib import resources from pathlib import Path from joblib import Parallel, delayed @@ -47,7 +19,6 @@ from SALib.sample.morris import sample from SALib.util import compute_groups_matrix, read_param_file -from definitions import PROJECT_ROOT, get_OUTPUT_PATH from temoa._internal.table_writer import TableWriter from temoa.core.config import TemoaConfig from temoa.data_io.hybrid_loader import HybridLoader @@ -55,8 +26,8 @@ logger = logging.getLogger(__name__) -solver_options_file = Path( - PROJECT_ROOT, 'temoa/extensions/method_of_morris/morris_solver_options.toml' +solver_options_file = ( + resources.files('temoa.extensions.method_of_morris') / 'morris_solver_options.toml' ) @@ -87,8 +58,9 @@ def __init__(self, config: TemoaConfig): # read in the options try: - with open(solver_options_file, 'rb') as f: - all_options = tomllib.load(f) + with resources.as_file(solver_options_file) as path: + with open(path, 'rb') as f: + all_options = tomllib.load(f) s_options = all_options.get(self.config.solver_name, {}) logger.info('Using solver options: %s', s_options) @@ -98,7 +70,7 @@ def __init__(self, config: TemoaConfig): # output handling self.verbose = False # for troubleshooting - self.mm_output_folder = get_OUTPUT_PATH() / 'MM_outputs' + self.mm_output_folder = self.config.output_path / 'MM_outputs' self.mm_output_folder.mkdir(exist_ok=True) self.param_file: Path = self.mm_output_folder / 'params.csv' diff --git a/temoa/extensions/modeling_to_generate_alternatives/mga_sequencer.py b/temoa/extensions/modeling_to_generate_alternatives/mga_sequencer.py index bda3cf22..2b9e49a4 100644 --- a/temoa/extensions/modeling_to_generate_alternatives/mga_sequencer.py +++ b/temoa/extensions/modeling_to_generate_alternatives/mga_sequencer.py @@ -1,30 +1,5 @@ """ -Tools for Energy Model Optimization and Analysis (Temoa): -An open source framework for energy systems optimization modeling - -Copyright (C) 2015, NC State University - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -A complete copy of the GNU General Public License v2 (GPLv2) is available -in LICENSE.txt. Users uncompressing this from an archive may not have -received this license file. If not, see . - - -Written by: J. F. Hyink -jeff@westernspark.us -https://westernspark.us -Created on: 4/15/24 - -The purpose of this module is to perform top-level control over an MGA model run +Performs top-level control over an MGA model run """ import logging @@ -35,6 +10,7 @@ from datetime import datetime from logging import getLogger from multiprocessing import Queue +from importlib import resources from pathlib import Path from queue import Empty @@ -43,7 +19,6 @@ from pyomo.dataportal import DataPortal from pyomo.opt import check_optimal_termination -from definitions import PROJECT_ROOT, get_OUTPUT_PATH from temoa._internal.run_actions import build_instance from temoa._internal.table_writer import TableWriter from temoa.components.costs import total_cost_rule @@ -58,8 +33,8 @@ logger = getLogger(__name__) -solver_options_path = Path( - PROJECT_ROOT, 'temoa/extensions/modeling_to_generate_alternatives/MGA_solver_options.toml' +solver_options_path = ( + resources.files('temoa.extensions.modeling_to_generate_alternatives') / 'MGA_solver_options.toml' ) @@ -88,8 +63,9 @@ def __init__(self, config: TemoaConfig): # read in the options try: - with open(solver_options_path, 'rb') as f: - all_options = tomllib.load(f) + with resources.as_file(solver_options_path) as path: + with open(path, 'rb') as f: + all_options = tomllib.load(f) s_options = all_options.get(self.config.solver_name, {}) logger.info('Using solver options: %s', s_options) @@ -208,6 +184,7 @@ def start(self): con=self.con, optimal_cost=tot_cost, cost_relaxation=self.cost_epsilon, + config=self.config, ) # 5. Set up the Workers @@ -224,7 +201,7 @@ def start(self): 'solver_options': self.worker_solver_options, } # construct path for the solver logs - s_path = Path(get_OUTPUT_PATH(), 'solver_logs') + s_path = self.config.output_path / 'solver_logs' if not s_path.exists(): s_path.mkdir() for _ in range(num_workers): diff --git a/temoa/extensions/modeling_to_generate_alternatives/tech_activity_vector_manager.py b/temoa/extensions/modeling_to_generate_alternatives/tech_activity_vector_manager.py index 0828dc3a..8e28f7a7 100644 --- a/temoa/extensions/modeling_to_generate_alternatives/tech_activity_vector_manager.py +++ b/temoa/extensions/modeling_to_generate_alternatives/tech_activity_vector_manager.py @@ -1,31 +1,3 @@ -""" -Tools for Energy Model Optimization and Analysis (Temoa): -An open source framework for energy systems optimization modeling - -Copyright (C) 2015, NC State University - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -A complete copy of the GNU General Public License v2 (GPLv2) is available -in LICENSE.txt. Users uncompressing this from an archive may not have -received this license file. If not, see . - - -Written by: J. F. Hyink -jeff@westernspark.us -https://westernspark.us -Created on: 4/16/24 - -""" - import queue import sqlite3 from collections import defaultdict @@ -38,7 +10,7 @@ from matplotlib import pyplot as plt from pyomo.core import Expression, Objective, Var, quicksum, value -from definitions import get_OUTPUT_PATH +from temoa.core.config import TemoaConfig from temoa.core.model import TemoaModel from temoa.extensions.modeling_to_generate_alternatives.hull import Hull from temoa.extensions.modeling_to_generate_alternatives.mga_constants import MgaWeighting @@ -73,12 +45,14 @@ def __init__( weighting: MgaWeighting, optimal_cost: float, cost_relaxation: float, + config: TemoaConfig, ): self.completed_solves = 0 self.conn = conn self.base_model = base_model self.optimal_cost = optimal_cost self.cost_relaxation = cost_relaxation + self.config = config self.generation_index = 1 # index of how many models generated to couple inputs-outputs # {category : [technology, ...]} @@ -362,7 +336,7 @@ def tracker(self): self.perf_data.update({len(self.hull_points): volume}) def finalize_tracker(self): - fout = Path(get_OUTPUT_PATH(), 'hull_performance.png') + fout = self.config.output_path / 'hull_performance.png' pts = sorted(self.perf_data.keys()) y = [self.perf_data[pt] for pt in pts] plt.plot(pts, y) diff --git a/temoa/extensions/monte_carlo/example_builds/scenario_analyzer.py b/temoa/extensions/monte_carlo/example_builds/scenario_analyzer.py index 9d6ba05f..ba55b0fb 100644 --- a/temoa/extensions/monte_carlo/example_builds/scenario_analyzer.py +++ b/temoa/extensions/monte_carlo/example_builds/scenario_analyzer.py @@ -1,43 +1,16 @@ """ -Tools for Energy Model Optimization and Analysis (Temoa): -An open source framework for energy systems optimization modeling - -Copyright (C) 2015, NC State University - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -A complete copy of the GNU General Public License v2 (GPLv2) is available -in LICENSE.txt. Users uncompressing this from an archive may not have -received this license file. If not, see . - - -Written by: J. F. Hyink -jeff@westernspark.us -https://westernspark.us -Created on: 11/11/24 - Simple analyzer--example only - """ from math import sqrt from pathlib import Path from sqlite3 import Connection +from importlib import resources from matplotlib import pyplot as plt -from definitions import PROJECT_ROOT - scenario_name = 'Purple Onion' # must match config file -db_path = Path(PROJECT_ROOT, 'data_files/example_dbs/utopia.sqlite') -with Connection(db_path) as conn: +db_resource = resources.files('data_files.example_dbs') / 'utopia.sqlite' +with resources.as_file(db_resource) as db_path, Connection(str(db_path)) as conn: cur = conn.cursor() obj_values = cur.execute( f"SELECT total_system_cost FROM output_objective WHERE scenario LIKE '{scenario_name}-%'" diff --git a/temoa/extensions/monte_carlo/example_builds/scenario_maker.py b/temoa/extensions/monte_carlo/example_builds/scenario_maker.py index 0768b3fd..7f621777 100644 --- a/temoa/extensions/monte_carlo/example_builds/scenario_maker.py +++ b/temoa/extensions/monte_carlo/example_builds/scenario_maker.py @@ -1,28 +1,4 @@ """ -Tools for Energy Model Optimization and Analysis (Temoa): -An open source framework for energy systems optimization modeling - -Copyright (C) 2015, NC State University - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -A complete copy of the GNU General Public License v2 (GPLv2) is available -in LICENSE.txt. Users uncompressing this from an archive may not have -received this license file. If not, see . - - -Written by: J. F. Hyink -jeff@westernspark.us -https://westernspark.us -Created on: 11/11/24 This file is intended to be a simple EXAMPLE for testing mainly of how one might make a set of runs for a Monte Carlo simulation. @@ -47,8 +23,6 @@ import matplotlib.pyplot as plt import numpy as np -from definitions import PROJECT_ROOT - # distro for the related cost vars # multivariate norm generator: @@ -69,7 +43,7 @@ nuc_dev = np.random.binomial(n=1, p=0.20, size=num_runs) * -0.4 # put it together... -file_loc = Path(PROJECT_ROOT) / 'data_files/monte_carlo/run_settings_2.csv' +file_loc = Path.cwd() / 'run_settings_2.csv' with open(file_loc, 'w') as f: f.write('run,param,index,mod,value,notes\n') for run_idx in range(num_runs): diff --git a/temoa/extensions/monte_carlo/mc_run.py b/temoa/extensions/monte_carlo/mc_run.py index 537bea1f..a57ba3c9 100644 --- a/temoa/extensions/monte_carlo/mc_run.py +++ b/temoa/extensions/monte_carlo/mc_run.py @@ -1,28 +1,4 @@ """ -Tools for Energy Model Optimization and Analysis (Temoa): -An open source framework for energy systems optimization modeling - -Copyright (C) 2015, NC State University - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -A complete copy of the GNU General Public License v2 (GPLv2) is available -in LICENSE.txt. Users uncompressing this from an archive may not have -received this license file. If not, see . - - -Written by: J. F. Hyink -jeff@westernspark.us -https://westernspark.us -Created on: 11/9/24 """ @@ -34,7 +10,6 @@ from pyomo.dataportal import DataPortal -from definitions import PROJECT_ROOT from temoa.core.config import TemoaConfig from temoa.core.model import TemoaModel from temoa.data_io.hybrid_loader import HybridLoader @@ -222,14 +197,14 @@ def __init__(self, config: TemoaConfig, data_store: dict): self.config = config self.data_store = data_store self.tweak_factory = TweakFactory(data_store) - self.settings_file = PROJECT_ROOT / Path(self.config.monte_carlo_inputs['run_settings']) + self.settings_file = Path(self.config.monte_carlo_inputs['run_settings']) def prescreen_input_file(self): """ read the input csv file and screen common errors :return: True if file passes, false otherwise with log entries """ - with open(self.settings_file, 'r') as f: + with open(self.settings_file) as f: header = f.readline().strip() assert header == 'run,param,index,mod,value,notes', ( 'header should be: run,param,index,mod,value,notes' @@ -252,7 +227,7 @@ def _next_row_generator(self) -> Generator[tuple[int, str], None, None]: A generator to read lines from thr run settings file :return: """ - with open(self.settings_file, 'r') as f: + with open(self.settings_file) as f: # burn header f.readline() idx = 2 @@ -314,7 +289,7 @@ def element_locator(data_store: dict, param: str, target_index: tuple) -> list[t matches = [ k for k in raw_indices - if all((k[idx] == target_index[idx] for idx in non_wildcard_locs)) + if all(k[idx] == target_index[idx] for idx in non_wildcard_locs) ] return matches diff --git a/temoa/extensions/monte_carlo/mc_sequencer.py b/temoa/extensions/monte_carlo/mc_sequencer.py index 44945f68..e5985179 100644 --- a/temoa/extensions/monte_carlo/mc_sequencer.py +++ b/temoa/extensions/monte_carlo/mc_sequencer.py @@ -1,31 +1,7 @@ """ -Tools for Energy Model Optimization and Analysis (Temoa): -An open source framework for energy systems optimization modeling - -Copyright (C) 2015, NC State University - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -A complete copy of the GNU General Public License v2 (GPLv2) is available -in LICENSE.txt. Users uncompressing this from an archive may not have -received this license file. If not, see . - - -Written by: J. F. Hyink -jeff@westernspark.us -https://westernspark.us -Created on: 11/9/24 A sequencer for Monte Carlo Runs -. + """ import logging @@ -36,11 +12,11 @@ from datetime import datetime from logging import getLogger from multiprocessing import Queue +from importlib import resources from pathlib import Path from pyomo.dataportal import DataPortal -from definitions import PROJECT_ROOT, get_OUTPUT_PATH from temoa._internal.data_brick import DataBrick from temoa._internal.table_writer import TableWriter from temoa.core.config import TemoaConfig @@ -50,7 +26,9 @@ logger = getLogger(__name__) -solver_options_path = Path(PROJECT_ROOT, 'temoa/extensions/monte_carlo/MC_solver_options.toml') +solver_options_path = ( + resources.files('temoa.extensions.monte_carlo') / 'MC_solver_options.toml' +) class MCSequencer: @@ -63,8 +41,9 @@ def __init__(self, config: TemoaConfig): # read in the options try: - with open(solver_options_path, 'rb') as f: - all_options = tomllib.load(f) + with resources.as_file(solver_options_path) as path: + with open(path, 'rb') as f: + all_options = tomllib.load(f) s_options = all_options.get(self.config.solver_name, {}) logger.info('Using solver options: %s', s_options) @@ -130,7 +109,7 @@ def start(self): 'solver_options': self.worker_solver_options, } # construct path for the solver logs - s_path = Path(get_OUTPUT_PATH(), 'solver_logs') + s_path = self.config.output_path / 'solver_logs' if not s_path.exists(): s_path.mkdir() for i in range(num_workers): diff --git a/temoa/extensions/myopic/myopic_sequencer.py b/temoa/extensions/myopic/myopic_sequencer.py index 6e755814..c0877bf2 100644 --- a/temoa/extensions/myopic/myopic_sequencer.py +++ b/temoa/extensions/myopic/myopic_sequencer.py @@ -7,10 +7,10 @@ import sqlite3 import sys from collections import deque +from importlib import resources from pathlib import Path from sqlite3 import Connection -import definitions from temoa._internal import run_actions from temoa._internal.table_writer import TableWriter from temoa.core.config import TemoaConfig @@ -23,9 +23,7 @@ logger = logging.getLogger(__name__) -table_script_file = Path( - definitions.PROJECT_ROOT, 'temoa/extensions/myopic', 'make_myopic_tables.sql' -) +table_script_file = resources.files('temoa.extensions.myopic') / 'make_myopic_tables.sql' class MyopicSequencer: @@ -141,7 +139,8 @@ def start(self): self.characterize_run() # create the Myopic Output tables, if they don't already exist. - self.execute_script(table_script_file) + with resources.as_file(table_script_file) as script_path: + self.execute_script(script_path) # clear out the old riff-raff self.clear_old_results() diff --git a/tests/conftest.py b/tests/conftest.py index dbc332e3..0e405a52 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,7 +7,6 @@ import pytest from pyomo.opt import SolverResults -from definitions import PROJECT_ROOT, set_OUTPUT_PATH from temoa._internal.temoa_sequencer import TemoaSequencer from temoa.core.config import TemoaConfig from temoa.core.model import TemoaModel @@ -15,14 +14,14 @@ logger = logging.getLogger(__name__) # set the target folder for output from testing -output_path = os.path.join(PROJECT_ROOT, 'tests', 'testing_log') -if not os.path.exists(output_path): - os.mkdir(output_path) +output_path = Path(__file__).parent / 'testing_log' +if not output_path.exists(): + output_path.mkdir() # set up logger in conftest.py so that it is properly anchored in the test folder. filename = 'testing.log' logging.basicConfig( - filename=os.path.join(output_path, filename), + filename=output_path / filename, filemode='w', format='%(asctime)s | %(module)s | %(levelname)s | %(message)s', datefmt='%d-%b-%y %H:%M:%S', @@ -36,8 +35,8 @@ def refresh_databases() -> None: """make new databases from source for testing... removes possibility of contamination by earlier runs""" - data_output_path = Path(PROJECT_ROOT, 'tests', 'testing_outputs') - data_source_path = Path(PROJECT_ROOT, 'tests', 'testing_data') + data_output_path = Path(__file__).parent / 'testing_outputs' + data_source_path = Path(__file__).parent / 'testing_data' databases = ( ('utopia.sql', 'utopia.sqlite'), ('utopia.sql', 'myo_utopia.sqlite'), @@ -66,7 +65,7 @@ def refresh_databases() -> None: @pytest.fixture() def system_test_run( - request, tmp_path + request: Any, tmp_path: Path ) -> tuple[Any, SolverResults | None, TemoaModel | None, TemoaSequencer]: """ spin up the model, solve it, and hand over the model and result for inspection @@ -74,9 +73,7 @@ def system_test_run( data_name = request.param['name'] logger.info('Setting up and solving: %s', data_name) filename = request.param['filename'] - config_file = Path(PROJECT_ROOT, 'tests', 'testing_configs', filename) - - set_OUTPUT_PATH(tmp_path) + config_file = Path(__file__).parent / 'testing_configs' / filename config = TemoaConfig.build_config( config_file=config_file, diff --git a/tests/test_linked_tech.py b/tests/test_linked_tech.py index c174a05c..cfa66240 100644 --- a/tests/test_linked_tech.py +++ b/tests/test_linked_tech.py @@ -1,28 +1,4 @@ """ -Tools for Energy Model Optimization and Analysis (Temoa): -An open source framework for energy systems optimization modeling - -Copyright (C) 2015, NC State University - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -A complete copy of the GNU General Public License v2 (GPLv2) is available -in LICENSE.txt. Users uncompressing this from an archive may not have -received this license file. If not, see . - - -Written by: J. F. Hyink -jeff@westernspark.us -https://westernspark.us -Created on: 6/22/24 A quick test on Linked Tech. The scenario is described in an image in the testing_data folder: simple_linked_tech_description.jpg @@ -35,8 +11,6 @@ import pytest -from definitions import PROJECT_ROOT - logger = logging.getLogger(__name__) config_files = [ {'name': 'link', 'filename': 'config_link_test.toml'}, @@ -53,7 +27,7 @@ def test_linked_tech(system_test_run): """Check a few known values. See the note above in header regarding scenario reference""" data_name, res, mdl, _ = system_test_run # test emission of CO2 - output_db_path = Path(PROJECT_ROOT, 'tests', 'testing_outputs', 'simple_linked_tech.sqlite') + output_db_path = Path(__file__).parent / 'testing_outputs' / 'simple_linked_tech.sqlite' print(output_db_path) conn = sqlite3.connect(str(output_db_path)) co2_emiss = conn.execute( diff --git a/tests/test_material_results.py b/tests/test_material_results.py index 1c00e8cf..c292ac57 100644 --- a/tests/test_material_results.py +++ b/tests/test_material_results.py @@ -1,10 +1,11 @@ import logging import sqlite3 +from collections.abc import Generator from pathlib import Path +from typing import Any import pytest -from definitions import PROJECT_ROOT from temoa._internal.temoa_sequencer import TemoaSequencer from temoa.core.config import TemoaConfig @@ -12,14 +13,16 @@ @pytest.fixture(scope='module') -def solved_connection(request, tmp_path_factory): +def solved_connection( + request: Any, tmp_path_factory: Any +) -> Generator[tuple[sqlite3.Connection, str, str, int, float], None, None]: """ spin up the model, solve it, and hand over a connection to the results db """ data_name = 'materials' logger.info('Setting up and solving: %s', data_name) filename = 'config_materials.toml' - config_file = Path(PROJECT_ROOT, 'tests', 'testing_configs', filename) + config_file = Path(__file__).parent / 'testing_configs' / filename tmp_path = tmp_path_factory.mktemp('data') config = TemoaConfig.build_config(config_file=config_file, output_path=tmp_path, silent=True) sequencer = TemoaSequencer(config=config) @@ -51,9 +54,9 @@ def solved_connection(request, tmp_path_factory): 'solved_connection', argvalues=flow_tests, indirect=True, - ids=[t['name'] for t in flow_tests], + ids=[str(t['name']) for t in flow_tests], ) -def test_flows(solved_connection): +def test_flows(solved_connection: tuple[sqlite3.Connection, str, str, int, float]) -> None: """ Test that the emissions from each technology archetype are correct, and check total emissions """ diff --git a/tests/test_model.py b/tests/test_model.py index 35cfaf6f..5692333d 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -5,20 +5,19 @@ import pathlib import pickle -from definitions import PROJECT_ROOT from temoa._internal.temoa_sequencer import TemoaSequencer from temoa.core.config import TemoaConfig from temoa.core.modes import TemoaMode -def test_serialization(): +def test_serialization() -> None: """ Test to ensure the model pickles properly. This is used when employing mpi4py which requires that jobs passed are pickle-able. """ config_filename = 'config_utopia.toml' - config_file_path = pathlib.Path(PROJECT_ROOT, 'tests', 'testing_configs', config_filename) - output_path = pathlib.Path(PROJECT_ROOT, 'tests', 'testing_outputs') + config_file_path = pathlib.Path(__file__).parent / 'testing_configs' / config_filename + output_path = pathlib.Path(__file__).parent / 'testing_outputs' config = TemoaConfig.build_config( config_file=config_file_path, output_path=output_path, silent=True diff --git a/tests/test_set_consistency.py b/tests/test_set_consistency.py index e92efbe7..52444914 100644 --- a/tests/test_set_consistency.py +++ b/tests/test_set_consistency.py @@ -9,7 +9,6 @@ import pytest from pyomo import environ as pyo -from definitions import PROJECT_ROOT from temoa._internal.temoa_sequencer import TemoaSequencer from temoa.core.config import TemoaConfig from temoa.core.modes import TemoaMode @@ -44,7 +43,7 @@ def test_set_consistency(data_name, config_file, set_file, tmp_path): model_sets = {k: set(v) for k, v in model_sets.items()} # retrieve the cache and convert the set values from list -> set (json can't store sets) - cache_file = pathlib.Path(PROJECT_ROOT, 'tests', 'testing_data', set_file) + cache_file = pathlib.Path(__file__).parent / 'testing_data' / set_file with open(cache_file) as src: cached_sets = json.load(src) cached_sets = { diff --git a/tests/utilities/capture_set_sizes_for_cache.py b/tests/utilities/capture_set_sizes_for_cache.py index 900d4536..1536ff45 100644 --- a/tests/utilities/capture_set_sizes_for_cache.py +++ b/tests/utilities/capture_set_sizes_for_cache.py @@ -1,29 +1,5 @@ """ Utility to capture the set sizes for inspection/comparison - -Written by: J. F. Hyink -jeff@westernspark.us -https://westernspark.us -Created on: 11/14/23 - -Tools for Energy Model Optimization and Analysis (Temoa): -An open source framework for energy systems optimization modeling - -Copyright (C) 2015, NC State University - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -A complete copy of the GNU General Public License v2 (GPLv2) is available -in LICENSE.txt. Users uncompressing this from an archive may not have -received this license file. If not, see . """ import json @@ -33,8 +9,8 @@ import pyomo.environ as pyo -from definitions import PROJECT_ROOT from temoa._internal.temoa_sequencer import TemoaSequencer +from temoa.core.config import TemoaConfig logger = logging.getLogger(__name__) @@ -47,13 +23,17 @@ t = input('Type "Y" to continue, any other key to exit now.') if t not in {'y', 'Y'}: sys.exit(0) -output_file = Path(PROJECT_ROOT, 'tests', 'testing_data', 'US_9R_8D_set_sizes.json') -config_file = Path(PROJECT_ROOT, 'tests', 'utilities', 'config_US_9R_8D.toml') -options = {'silent': True, 'debug': True} -sequencer = TemoaSequencer( - config_file=config_file, output_path=Path(PROJECT_ROOT, 'tests', 'testing_log'), **options +output_file = Path(__file__).parent.parent / 'testing_data' / 'US_9R_8D_set_sizes.json' +config_file_path = Path(__file__).parent / 'config_US_9R_8D.toml' +output_path = Path(__file__).parent.parent / 'testing_log' +output_path.mkdir(parents=True, exist_ok=True) + +options = {'silent': True} +config = TemoaConfig.build_config( + config_file=config_file_path, output_path=output_path, silent=options['silent'] ) -instance = sequencer.start() +sequencer = TemoaSequencer(config=config) +instance = sequencer.build_model() # catch the built model model_sets = instance.component_map(ctype=pyo.Set) sets_dict = {k: len(v) for k, v in model_sets.items() if '_index' not in k} diff --git a/tests/utilities/capture_set_values_for_cache.py b/tests/utilities/capture_set_values_for_cache.py index 36926f08..3c5eed98 100644 --- a/tests/utilities/capture_set_values_for_cache.py +++ b/tests/utilities/capture_set_values_for_cache.py @@ -2,30 +2,6 @@ Quick utility to capture set values from a pyomo model to enable later comparison. This file should not need to be run again unless model schema changes - -Written by: J. F. Hyink -jeff@westernspark.us -https://westernspark.us -Created on: 8/26/23 - -Tools for Energy Model Optimization and Analysis (Temoa): -An open source framework for energy systems optimization modeling - -Copyright (C) 2015, NC State University - -This program is free software; you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation; either version 2 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -A complete copy of the GNU General Public License v2 (GPLv2) is available -in LICENSE.txt. Users uncompressing this from an archive may not have -received this license file. If not, see . """ import json @@ -34,8 +10,8 @@ import pyomo.environ as pyo -from definitions import PROJECT_ROOT from temoa._internal.temoa_sequencer import TemoaSequencer +from temoa.core.config import TemoaConfig from tests.conftest import refresh_databases print( @@ -52,29 +28,33 @@ if t not in {'y', 'Y'}: sys.exit(0) -output_path = Path(PROJECT_ROOT, 'tests', 'testing_log') # capture the log here +output_path = Path(__file__).parent.parent / 'testing_log' # capture the log here +output_path.mkdir(parents=True, exist_ok=True) scenarios = [ { - 'output_file': Path(PROJECT_ROOT, 'tests', 'testing_data', 'utopia_sets.json'), - 'config_file': Path(PROJECT_ROOT, 'tests', 'utilities', 'config_utopia.toml'), + 'output_file': Path(__file__).parent.parent / 'testing_data' / 'utopia_sets.json', + 'config_file': Path(__file__).parent / 'config_utopia.toml', }, { - 'output_file': Path(PROJECT_ROOT, 'tests', 'testing_data', 'test_system_sets.json'), - 'config_file': Path(PROJECT_ROOT, 'tests', 'utilities', 'config_test_system.toml'), + 'output_file': Path(__file__).parent.parent / 'testing_data' / 'test_system_sets.json', + 'config_file': Path(__file__).parent / 'config_test_system.toml', }, { - 'output_file': Path(PROJECT_ROOT, 'tests', 'testing_data', 'mediumville_sets.json'), - 'config_file': Path(PROJECT_ROOT, 'tests', 'utilities', 'config_mediumville.toml'), + 'output_file': Path(__file__).parent.parent / 'testing_data' / 'mediumville_sets.json', + 'config_file': Path(__file__).parent / 'config_mediumville.toml', }, ] # make new copies of the DB's from source... refresh_databases() for scenario in scenarios: - ts = TemoaSequencer(config_file=scenario['config_file'], output_path=output_path) + config = TemoaConfig.build_config( + config_file=scenario['config_file'], output_path=output_path, silent=True + ) + ts = TemoaSequencer(config=config) - built_instance = ts.start() # catch the built model + built_instance = ts.build_model() # catch the built model model_sets = built_instance.component_map(ctype=pyo.Set) sets_dict = {k: list(v) for k, v in model_sets.items()}