From 33485971e1e3c108431b23f2439129fb490e411a Mon Sep 17 00:00:00 2001 From: Ross Whitfield Date: Thu, 17 Nov 2022 14:01:50 -0500 Subject: [PATCH] Fix runspaceInitComponent when using multiple configurations --- ipsframework/configurationManager.py | 9 ++- ipsframework/runspaceInitComponent.py | 79 +++------------------------ tests/multirun/test_basic_serial.py | 19 ++++--- 3 files changed, 22 insertions(+), 85 deletions(-) diff --git a/ipsframework/configurationManager.py b/ipsframework/configurationManager.py index ef13e9e6..4751d55d 100644 --- a/ipsframework/configurationManager.py +++ b/ipsframework/configurationManager.py @@ -443,11 +443,6 @@ def _initialize_sim(self, sim_data): if 'IPS_ROOT' not in comp_conf: if 'IPS_ROOT' in sim_conf: comp_conf['IPS_ROOT'] = sim_conf['IPS_ROOT'] - if 'DATA_TREE_ROOT' not in comp_conf: - if 'DATA_TREE_ROOT' in sim_conf: - comp_conf['DATA_TREE_ROOT'] = sim_conf['DATA_TREE_ROOT'] - else: - comp_conf['DATA_TREE_ROOT'] = sim_data.conf_file_dir if 'BIN_DIR' not in comp_conf: if 'BIN_DIR' in sim_conf: comp_conf['BIN_DIR'] = sim_conf['BIN_DIR'] @@ -552,6 +547,10 @@ def get_all_simulation_components_map(self): del sim_comps[self.fwk_sim_name] return sim_comps + def get_all_simulation_sim_root(self): + sim_roots = {name: sim_map.sim_root for name, sim_map in self.sim_map.items()} + return sim_roots + def get_framework_components(self): """ Return list of framework components. diff --git a/ipsframework/runspaceInitComponent.py b/ipsframework/runspaceInitComponent.py index dc22f1db..ca5bffc8 100644 --- a/ipsframework/runspaceInitComponent.py +++ b/ipsframework/runspaceInitComponent.py @@ -6,18 +6,6 @@ from ipsframework import ipsutil -def catch_and_go(func_to_decorate): - def new_func(*original_args, **original_kwargs): - # Do whatever else you want here - obj = original_args[0] - try: - func_to_decorate(*original_args, **original_kwargs) - except Exception as e: - obj.services.exception("Exception in call to %s:%s" % (obj.__class__.__name__, func_to_decorate.__name__)) - print(e) - return new_func - - class runspaceInitComponent(Component): """ Framework component to manage runspace initialization, container file @@ -34,33 +22,16 @@ def __init__(self, services, config): self.simRootDir = services.get_config_param('SIM_ROOT') self.cwd = self.config['OS_CWD'] - @catch_and_go def init(self, timestamp=0.0, **keywords): """ Creates base directory, copies IPS and FacetsComposer input files. """ - services = self.services - - try: - os.chdir(self.cwd) - except OSError: - self.services.debug('Working directory %s does not exist - this is impossibile', - self.cwd) - raise - if not self.simRootDir.startswith("/"): self.simRootDir = os.path.join(self.cwd, self.simRootDir) - # try making the simulation root directory - try: - os.makedirs(self.simRootDir, exist_ok=True) - except OSError as oserr: - self.services.exception('Error creating directory %s : %s', - self.simRootDir, oserr.strerror) - - config_files = services.fwk.config_file_list - platform_file = services.fwk.platform_file_name + config_files = self.services.fwk.config_file_list + platform_file = self.services.fwk.platform_file_name # Determine where the file is...if there's not an absolute path specified, # assume that it was in the directory that the IPS was launched from. @@ -79,29 +50,19 @@ def init(self, timestamp=0.0, **keywords): ipsutil.copyFiles(conf_file_loc, config_files, self.simRootDir) ipsutil.copyFiles(plat_file_loc, platform_file, self.simRootDir) - @catch_and_go def step(self, timestamp=0.0, **keywords): """ Copies individual subcomponent input files into working subdirectories. """ - services = self.services - - # sim_comps = services.fwk.config_manager.get_component_map() - sim_comps = services.fwk.config_manager.get_all_simulation_components_map() - registry = services.fwk.comp_registry + sim_comps = self.services.fwk.config_manager.get_all_simulation_components_map() + sim_roots = self.services.fwk.config_manager.get_all_simulation_sim_root() + registry = self.services.fwk.comp_registry simulation_setup = os.path.join(self.simRootDir, 'simulation_setup') - # make the simulation_setup directory for scripts - try: - os.makedirs(simulation_setup, exist_ok=True) - except OSError as oserr: - self.services.exception('Error creating directory %s : %s', - simulation_setup, oserr.strerror) - # for each simulation component - for comp_list in sim_comps.values(): + for name, comp_list in sim_comps.items(): # for each component_id in the list of components for comp_id in comp_list: # build the work directory name @@ -111,7 +72,7 @@ def step(self, timestamp=0.0, **keywords): str(comp_id.get_seq_num())]) # compose the workdir name - workdir = os.path.join(self.simRootDir, 'work', full_comp_id) + workdir = os.path.join(sim_roots[name], 'work', full_comp_id) # make the working directory try: @@ -130,21 +91,6 @@ def step(self, timestamp=0.0, **keywords): print('Error copying input files for initialization') raise - # This is a bit tricky because we want to look either in the same - # place as the input files or the data_tree root - if 'DATA_FILES' in comp_conf: - filesCopied = False - if 'DATA_TREE_ROOT' in comp_conf: - dtrdir = os.path.abspath(comp_conf['DATA_TREE_ROOT']) - if os.path.exists(os.path.join(dtrdir, comp_conf['DATA_FILES'][0])): - ipsutil.copyFiles(dtrdir, os.path.basename(comp_conf['DATA_FILES']), - workdir) - filesCopied = True - if not filesCopied: - ipsutil.copyFiles(os.path.abspath(comp_conf['INPUT_DIR']), - os.path.basename(comp_conf['DATA_FILES']), - workdir) - # copy the component's script to the simulation_setup directory if comp_conf['SCRIPT']: if os.path.isabs(comp_conf['SCRIPT']): @@ -155,14 +101,3 @@ def step(self, timestamp=0.0, **keywords): ipsutil.copyFiles(comp_conf['BIN_DIR'], [os.path.basename(comp_conf['SCRIPT'])], simulation_setup) - - # get the working directory from the runspaceInitComponent - workdir = services.get_working_dir() - - # create the working directory for this component - try: - os.makedirs(workdir, exist_ok=True) - except OSError as oserr: - self.services.exception('Error creating directory %s : %s', - workdir, oserr.strerror) - raise diff --git a/tests/multirun/test_basic_serial.py b/tests/multirun/test_basic_serial.py index 2ef48423..ee9f0351 100644 --- a/tests/multirun/test_basic_serial.py +++ b/tests/multirun/test_basic_serial.py @@ -139,11 +139,8 @@ def test_basic_serial_multi(tmpdir, capfd): # check files copied and created for no in ["1", "2"]: - # This should also work for 2 - if no == "2": - continue - driver_files = [os.path.basename(f) for f in glob.glob(str(tmpdir.join(f"test_basic_serial{no}_0/work/drivers_testing_basic_serial1_*/*")))] + driver_files = [os.path.basename(f) for f in glob.glob(str(tmpdir.join(f"test_basic_serial{no}_0/work/drivers_testing_basic_serial*_*/*")))] for infile in ["file1", "ofile1", "ofile2", "sfile1", "sfile2"]: assert infile in driver_files @@ -151,10 +148,16 @@ def test_basic_serial_multi(tmpdir, capfd): medium_worker_files = [os.path.basename(f) for f in glob.glob(str(tmpdir.join(f"test_basic_serial{no}_0/work/workers_testing_medium_worker_*/*")))] large_worker_files = [os.path.basename(f) for f in glob.glob(str(tmpdir.join(f"test_basic_serial{no}_0/work/workers_testing_large_worker_*/*")))] - for outfile in ["my_out3.50", "my_out3.60", "my_out3.70"]: - assert outfile in small_worker_files - assert outfile in medium_worker_files - assert outfile in large_worker_files + if no == "1": + for outfile in ["my_out3.50", "my_out3.60", "my_out3.70"]: + assert outfile in small_worker_files + assert outfile in medium_worker_files + assert outfile in large_worker_files + else: + for outfile in ["my_out3.40", "my_out3.50", "my_out3.60"]: + assert outfile in small_worker_files + assert outfile in medium_worker_files + assert outfile in large_worker_files # check contents of my_out files for outfile in ["my_out3.50", "my_out3.60", "my_out3.70"]: