Skip to content

Fix runspaceInitComponent when using multiple configurations #178

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rosswhitfield merged 1 commit into from
Nov 17, 2022
Merged

Fix runspaceInitComponent when using multiple configurations #178

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 4 additions & 5 deletions ipsframework/configurationManager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -443,11 +443,6 @@ def _initialize_sim(self, sim_data):
if 'IPS_ROOT' not in comp_conf:
if 'IPS_ROOT' in sim_conf:
comp_conf['IPS_ROOT'] = sim_conf['IPS_ROOT']
if 'DATA_TREE_ROOT' not in comp_conf:
if 'DATA_TREE_ROOT' in sim_conf:
comp_conf['DATA_TREE_ROOT'] = sim_conf['DATA_TREE_ROOT']
else:
comp_conf['DATA_TREE_ROOT'] = sim_data.conf_file_dir
if 'BIN_DIR' not in comp_conf:
if 'BIN_DIR' in sim_conf:
comp_conf['BIN_DIR'] = sim_conf['BIN_DIR']
Expand Down Expand Up @@ -552,6 +547,10 @@ def get_all_simulation_components_map(self):
del sim_comps[self.fwk_sim_name]
return sim_comps

def get_all_simulation_sim_root(self):
sim_roots = {name: sim_map.sim_root for name, sim_map in self.sim_map.items()}
return sim_roots

def get_framework_components(self):
"""
Return list of framework components.
Expand Down
79 changes: 7 additions & 72 deletions ipsframework/runspaceInitComponent.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,6 @@
from ipsframework import ipsutil


def catch_and_go(func_to_decorate):
def new_func(*original_args, **original_kwargs):
# Do whatever else you want here
obj = original_args[0]
try:
func_to_decorate(*original_args, **original_kwargs)
except Exception as e:
obj.services.exception("Exception in call to %s:%s" % (obj.__class__.__name__, func_to_decorate.__name__))
print(e)
return new_func


class runspaceInitComponent(Component):
"""
Framework component to manage runspace initialization, container file
Expand All @@ -34,33 +22,16 @@ def __init__(self, services, config):
self.simRootDir = services.get_config_param('SIM_ROOT')
self.cwd = self.config['OS_CWD']

@catch_and_go
def init(self, timestamp=0.0, **keywords):
"""
Creates base directory, copies IPS and FacetsComposer input files.
"""

services = self.services

try:
os.chdir(self.cwd)
except OSError:
self.services.debug('Working directory %s does not exist - this is impossibile',
self.cwd)
raise

if not self.simRootDir.startswith("/"):
self.simRootDir = os.path.join(self.cwd, self.simRootDir)

# try making the simulation root directory
try:
os.makedirs(self.simRootDir, exist_ok=True)
except OSError as oserr:
self.services.exception('Error creating directory %s : %s',
self.simRootDir, oserr.strerror)

config_files = services.fwk.config_file_list
platform_file = services.fwk.platform_file_name
config_files = self.services.fwk.config_file_list
platform_file = self.services.fwk.platform_file_name

# Determine where the file is...if there's not an absolute path specified,
# assume that it was in the directory that the IPS was launched from.
Expand All @@ -79,29 +50,19 @@ def init(self, timestamp=0.0, **keywords):
ipsutil.copyFiles(conf_file_loc, config_files, self.simRootDir)
ipsutil.copyFiles(plat_file_loc, platform_file, self.simRootDir)

@catch_and_go
def step(self, timestamp=0.0, **keywords):
"""
Copies individual subcomponent input files into working subdirectories.
"""

services = self.services

# sim_comps = services.fwk.config_manager.get_component_map()
sim_comps = services.fwk.config_manager.get_all_simulation_components_map()
registry = services.fwk.comp_registry
sim_comps = self.services.fwk.config_manager.get_all_simulation_components_map()
sim_roots = self.services.fwk.config_manager.get_all_simulation_sim_root()
registry = self.services.fwk.comp_registry

simulation_setup = os.path.join(self.simRootDir, 'simulation_setup')

# make the simulation_setup directory for scripts
try:
os.makedirs(simulation_setup, exist_ok=True)
except OSError as oserr:
self.services.exception('Error creating directory %s : %s',
simulation_setup, oserr.strerror)

# for each simulation component
for comp_list in sim_comps.values():
for name, comp_list in sim_comps.items():
# for each component_id in the list of components
for comp_id in comp_list:
# build the work directory name
Expand All @@ -111,7 +72,7 @@ def step(self, timestamp=0.0, **keywords):
str(comp_id.get_seq_num())])

# compose the workdir name
workdir = os.path.join(self.simRootDir, 'work', full_comp_id)
workdir = os.path.join(sim_roots[name], 'work', full_comp_id)

# make the working directory
try:
Expand All @@ -130,21 +91,6 @@ def step(self, timestamp=0.0, **keywords):
print('Error copying input files for initialization')
raise

# This is a bit tricky because we want to look either in the same
# place as the input files or the data_tree root
if 'DATA_FILES' in comp_conf:
filesCopied = False
if 'DATA_TREE_ROOT' in comp_conf:
dtrdir = os.path.abspath(comp_conf['DATA_TREE_ROOT'])
if os.path.exists(os.path.join(dtrdir, comp_conf['DATA_FILES'][0])):
ipsutil.copyFiles(dtrdir, os.path.basename(comp_conf['DATA_FILES']),
workdir)
filesCopied = True
if not filesCopied:
ipsutil.copyFiles(os.path.abspath(comp_conf['INPUT_DIR']),
os.path.basename(comp_conf['DATA_FILES']),
workdir)

# copy the component's script to the simulation_setup directory
if comp_conf['SCRIPT']:
if os.path.isabs(comp_conf['SCRIPT']):
Expand All @@ -155,14 +101,3 @@ def step(self, timestamp=0.0, **keywords):
ipsutil.copyFiles(comp_conf['BIN_DIR'],
[os.path.basename(comp_conf['SCRIPT'])],
simulation_setup)

# get the working directory from the runspaceInitComponent
workdir = services.get_working_dir()

# create the working directory for this component
try:
os.makedirs(workdir, exist_ok=True)
except OSError as oserr:
self.services.exception('Error creating directory %s : %s',
workdir, oserr.strerror)
raise
19 changes: 11 additions & 8 deletions tests/multirun/test_basic_serial.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,22 +139,25 @@ def test_basic_serial_multi(tmpdir, capfd):

# check files copied and created
for no in ["1", "2"]:
# This should also work for 2
if no == "2":
continue

driver_files = [os.path.basename(f) for f in glob.glob(str(tmpdir.join(f"test_basic_serial{no}_0/work/drivers_testing_basic_serial1_*/*")))]
driver_files = [os.path.basename(f) for f in glob.glob(str(tmpdir.join(f"test_basic_serial{no}_0/work/drivers_testing_basic_serial*_*/*")))]
for infile in ["file1", "ofile1", "ofile2", "sfile1", "sfile2"]:
assert infile in driver_files

small_worker_files = [os.path.basename(f) for f in glob.glob(str(tmpdir.join(f"test_basic_serial{no}_0/work/workers_testing_small_worker_*/*")))]
medium_worker_files = [os.path.basename(f) for f in glob.glob(str(tmpdir.join(f"test_basic_serial{no}_0/work/workers_testing_medium_worker_*/*")))]
large_worker_files = [os.path.basename(f) for f in glob.glob(str(tmpdir.join(f"test_basic_serial{no}_0/work/workers_testing_large_worker_*/*")))]

for outfile in ["my_out3.50", "my_out3.60", "my_out3.70"]:
assert outfile in small_worker_files
assert outfile in medium_worker_files
assert outfile in large_worker_files
if no == "1":
for outfile in ["my_out3.50", "my_out3.60", "my_out3.70"]:
assert outfile in small_worker_files
assert outfile in medium_worker_files
assert outfile in large_worker_files
else:
for outfile in ["my_out3.40", "my_out3.50", "my_out3.60"]:
assert outfile in small_worker_files
assert outfile in medium_worker_files
assert outfile in large_worker_files

# check contents of my_out files
for outfile in ["my_out3.50", "my_out3.60", "my_out3.70"]:
Expand Down