From afd9d8fa25e2524bbc723050baa5451b06e7a7fc Mon Sep 17 00:00:00 2001 From: Zak V Date: Mon, 2 Nov 2020 14:19:03 -0500 Subject: [PATCH 01/22] Simplified Run.group's behavior when initialized outside of a lyse script. --- lyse/__init__.py | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/lyse/__init__.py b/lyse/__init__.py index 67ce126..86be823 100644 --- a/lyse/__init__.py +++ b/lyse/__init__.py @@ -104,7 +104,7 @@ def globals_diff(run1, run2, group=None): class Run(object): def __init__(self,h5_path,no_write=False): self.no_write = no_write - self._no_group = None + self.group = None self.h5_path = h5_path if not self.no_write: self._create_group_if_not_exists(h5_path, '/', 'results') @@ -116,21 +116,17 @@ def __init__(self,h5_path,no_write=False): # this Run object: frame = inspect.currentframe() __file__ = frame.f_back.f_globals['__file__'] - self.group = os.path.basename(__file__).split('.py')[0] - self._create_group_if_not_exists(h5_path, 'results', self.group) + group = os.path.basename(__file__).split('.py')[0] + self._create_group_if_not_exists(h5_path, 'results', group) + self.group = group except KeyError: # sys.stderr.write('Warning: to write results, call ' # 'Run.set_group(groupname), specifying the name of the group ' # 'you would like to save results to. This normally comes from ' # 'the filename of your script, but since you\'re in interactive ' - # 'mode, there is no scipt name. Opening in read only mode for ' - # 'the moment.\n') - - # Backup the value of self.no_write for restoration once the group - # is set - self._no_group = (True, self.no_write) - self.no_write = True - + # 'mode, there is no scipt name.\n') + pass + def _create_group_if_not_exists(self, h5_path, location, groupname): """Creates a group in the HDF5 file at `location` if it does not exist. @@ -142,16 +138,15 @@ def _create_group_if_not_exists(self, h5_path, location, groupname): if not groupname in h5_file[location]: create_group = True if create_group: + if self.no_write: + msg = "Cannot create group; this run is read-only." + raise PermissionError(msg) with h5py.File(h5_path, 'r+') as h5_file: h5_file[location].create_group(groupname) def set_group(self, groupname): + self._create_group_if_not_exists(self.h5_path, '/results', groupname) self.group = groupname - self._create_group_if_not_exists(self.h5_path, '/results', self.group) - # restore no_write attribute now we have set the group - if self._no_group is not None and self._no_group[0]: - self.no_write = self._no_group[1] - self._no_group = None def trace_names(self): with h5py.File(self.h5_path, 'r') as h5_file: @@ -213,6 +208,12 @@ def save_result(self, name, value, group=None, overwrite=True): 'single Run object is used') with h5py.File(self.h5_path,'a') as h5_file: if not group: + if self.group is None: + msg = """Cannot save result; no default group set. Either + specify a value for this method's optional group + argument, or set a default value using the set_group() + method.""" + raise ValueError(dedent(msg)) # Save to analysis results group by default group = 'results/' + self.group elif not group in h5_file: @@ -244,6 +245,12 @@ def save_result_array(self, name, data, group=None, with h5py.File(self.h5_path, 'a') as h5_file: attrs = {} if not group: + if self.group is None: + msg = """Cannot save result; no default group set. Either + specify a value for this method's optional group + argument, or set a default value using the set_group() + method.""" + raise ValueError(dedent(msg)) # Save dataset to results group by default group = 'results/' + self.group elif not group in h5_file: From 029fdda57e27c019e36b5131736d0daceb768ff7 Mon Sep 17 00:00:00 2001 From: Zak V Date: Mon, 2 Nov 2020 15:08:02 -0500 Subject: [PATCH 02/22] Added docstring for Run.set_group(). --- lyse/__init__.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lyse/__init__.py b/lyse/__init__.py index 86be823..a457786 100644 --- a/lyse/__init__.py +++ b/lyse/__init__.py @@ -145,6 +145,18 @@ def _create_group_if_not_exists(self, h5_path, location, groupname): h5_file[location].create_group(groupname) def set_group(self, groupname): + """Set the default hdf5 file group for saving results. + + The `save...()` methods will save their results to `self.group` if an + explicit value for their optional `group` argument is not given. This + method updates `self.group`, making sure to create the group in the hdf5 + file if it does not already exist. + + Args: + groupname (str): The name of the hdf5 file group in which to save + results by default. The group will be created in the + `'/results'` group of the hdf5 file. + """ self._create_group_if_not_exists(self.h5_path, '/results', groupname) self.group = groupname From 565c1b3da4eb2b4bd35041f49220e12aef2d486b Mon Sep 17 00:00:00 2001 From: Zak V Date: Mon, 2 Nov 2020 15:45:35 -0500 Subject: [PATCH 03/22] Added a docstring for the Run class. --- lyse/__init__.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/lyse/__init__.py b/lyse/__init__.py index a457786..85d3bc2 100644 --- a/lyse/__init__.py +++ b/lyse/__init__.py @@ -102,6 +102,31 @@ def globals_diff(run1, run2, group=None): return dict_diff(run1.get_globals(group), run2.get_globals(group)) class Run(object): + """A class for saving/retrieving data to/from a run's hdf5 file. + + This class implements methods that allow the user to retrieve data from a + run's hdf5 file such as images, traces, and the values of globals. It also + provides methods for saving and retrieving results from analysis. + + Args: + h5_path (str): The path, including file name and extension, to the hdf5 + file for a run. + no_write (bool, optional): Set to `True` to prevent editing the run + file. Note that doing so prohibits the ability to save results to + the file. Defaults to `False`. + + Attributes: + h5_path (str): The value provided for `h5_path` during instantiation. + no_write (bool): The value provided for `no_write` during instantiation. + group (str): The group in the hdf5 file in which results are saved by + default. When a `Run` instance is created from within a lyse + singleshot or multishot routine, `group` will be set to the name of + the running routine. If created from outside a lyse script it will + be set to `None`. To change the default group for saving results, + use the `set_group()` method. Note that if `self.group` is `None` + and no value is provided for the optional `group` argument used by + the `save...()` methods, a `ValueError` will be raised. + """ def __init__(self,h5_path,no_write=False): self.no_write = no_write self.group = None From 997a6c70ea409e4252447bbb4c598f06421fb021 Mon Sep 17 00:00:00 2001 From: Zak V Date: Mon, 2 Nov 2020 15:50:52 -0500 Subject: [PATCH 04/22] Corrected some typos in strings in __init__.py. --- lyse/__init__.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lyse/__init__.py b/lyse/__init__.py index 85d3bc2..9ea312e 100644 --- a/lyse/__init__.py +++ b/lyse/__init__.py @@ -149,7 +149,7 @@ def __init__(self,h5_path,no_write=False): # 'Run.set_group(groupname), specifying the name of the group ' # 'you would like to save results to. This normally comes from ' # 'the filename of your script, but since you\'re in interactive ' - # 'mode, there is no scipt name.\n') + # 'mode, there is no script name.\n') pass def _create_group_if_not_exists(self, h5_path, location, groupname): @@ -202,16 +202,16 @@ def get_attrs(self, group): def get_trace(self,name): with h5py.File(self.h5_path, 'r') as h5_file: if not name in h5_file['data']['traces']: - raise Exception('The trace \'%s\' doesn not exist'%name) + raise Exception('The trace \'%s\' does not exist'%name) trace = h5_file['data']['traces'][name] return array(trace['t'],dtype=float),array(trace['values'],dtype=float) def get_result_array(self,group,name): with h5py.File(self.h5_path, 'r') as h5_file: if not group in h5_file['results']: - raise Exception('The result group \'%s\' doesn not exist'%group) + raise Exception('The result group \'%s\' does not exist'%group) if not name in h5_file['results'][group]: - raise Exception('The result array \'%s\' doesn not exist'%name) + raise Exception('The result array \'%s\' does not exist'%name) return array(h5_file['results'][group][name]) def get_result(self, group, name): @@ -321,7 +321,7 @@ def get_result_arrays(self, group, *names): def save_results(self, *args, **kwargs): """Iteratively call save_result() on multiple results. Assumes arguments are ordered such that each result to be saved is - preceeded by the name of the attribute to save it under. + preceded by the name of the attribute to save it under. Keywords arguments are passed to each call of save_result().""" names = args[::2] values = args[1::2] @@ -340,7 +340,7 @@ def save_results_dict(self, results_dict, uncertainties=False, **kwargs): def save_result_arrays(self, *args, **kwargs): """Iteratively call save_result_array() on multiple data sets. Assumes arguments are ordered such that each dataset to be saved is - preceeded by the name to save it as. + preceded by the name to save it as. All keyword arguments are passed to each call of save_result_array().""" names = args[::2] values = args[1::2] @@ -485,7 +485,7 @@ def __init__(self, h5_path, run_paths, no_write=False): 'Sequence.set_group(groupname), specifying the name of the group ' 'you would like to save results to. This normally comes from ' 'the filename of your script, but since you\'re in interactive ' - 'mode, there is no scipt name. Opening in read only mode for ' + 'mode, there is no script name. Opening in read only mode for ' 'the moment.\n') self.no_write = True From 9132cf3506ab0e8b82ac0069adea5a9242c756f2 Mon Sep 17 00:00:00 2001 From: Zak V Date: Mon, 2 Nov 2020 16:31:23 -0500 Subject: [PATCH 05/22] Run._create_group_if_not_exists() can now handle another thread/process creating the hdf5 group partway through its execution. --- lyse/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lyse/__init__.py b/lyse/__init__.py index 9ea312e..26eb253 100644 --- a/lyse/__init__.py +++ b/lyse/__init__.py @@ -167,7 +167,12 @@ def _create_group_if_not_exists(self, h5_path, location, groupname): msg = "Cannot create group; this run is read-only." raise PermissionError(msg) with h5py.File(h5_path, 'r+') as h5_file: - h5_file[location].create_group(groupname) + # Catch the ValueError raised if the group was created by + # something else between the check above and now. + try: + h5_file[location].create_group(groupname) + except ValueError: + pass def set_group(self, groupname): """Set the default hdf5 file group for saving results. From 5281dea7405b038b5740c88df983bdcb666339fd Mon Sep 17 00:00:00 2001 From: Zak V Date: Tue, 3 Nov 2020 14:24:43 -0500 Subject: [PATCH 06/22] Updated docstrings for Run.save_result() and Run.save_results(). --- lyse/__init__.py | 70 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 62 insertions(+), 8 deletions(-) diff --git a/lyse/__init__.py b/lyse/__init__.py index 26eb253..b46d807 100644 --- a/lyse/__init__.py +++ b/lyse/__init__.py @@ -238,10 +238,43 @@ def get_results(self, group, *names): return results def save_result(self, name, value, group=None, overwrite=True): - """Save a result to h5 file. Defaults are to save to the active group - in the 'results' group and overwrite an existing result. - Note that the result is saved as an attribute of 'results/group' and - overwriting attributes causes h5 file size bloat.""" + """Save a result to the hdf5 file. + + With the default argument values this method saves to `self.group` in + the `'/results'` group and overwrites any existing value. Note that the + result is saved as an attribute and overwriting attributes causes hdf5 + file size bloat. + + Args: + name (str): The name of the result. This will be the name of the + attribute added to the hdf5 file's group. + value (any): The value of the result, which will be saved as the + value of the hdf5 group's attribute set by `name`. However note + that when saving large arrays, it is better to use the + `self.save_result_array()` method which will store the results + as a dataset in the hdf5 file. + group (str, optional): The group in the hdf5 file to which the + result will be saved as an attribute. If set to `None`, then the + result will be saved to `self.group` in `'/results'`. Note that + if a value is passed for `group` here then it will NOT have + `'/result'` prepended to it. This is in contrast to using the + default group set with `self.set_group()`; when the default + group is set with that method it WILL have `'/results'` + prepended to it when before saving results. Defaults to `None`. + overwrite (bool, optional): Sets whether or not to overwrite the + previous value if the attribute already exists. If set to + `False` and the attribute already exists, an `Exception` is + raised. Defaults to `True`. + + Raises: + Exception: An `Exception` is raised if `self.no_write` is `True` + because saving the result would edit the file. + ValueError: A `ValueError` is raised if `self.group` is `None` and + no value is provided for `group` because the method then doesn't + know where to save the result. + Exception: An `Exception` is raised if an attribute with name `name` + already exists but `overwrite` is set to `False`. + """ if self.no_write: raise Exception('This run is read-only. ' 'You can\'t save results to runs through a ' @@ -324,10 +357,31 @@ def get_result_arrays(self, group, *names): return results def save_results(self, *args, **kwargs): - """Iteratively call save_result() on multiple results. - Assumes arguments are ordered such that each result to be saved is - preceded by the name of the attribute to save it under. - Keywords arguments are passed to each call of save_result().""" + """Save multiple results to the hdf5 file. + + This method Iteratively call `self.save_result()` on multiple results. + It assumes arguments are ordered such that each result to be saved is + preceded by the name of the attribute to save it under. Keywords + arguments are passed to each call of `self.save_result()`. + + Args: + *args: The names and values of results to be saved. The first entry + should be a string giving the name of the first result, and the + second entry should be the value for that result. After that, + an arbitrary number of additional pairs of result name strings + and values can be included, e.g. + `'name0', value0, 'name1', value1,...`. + **kwargs: Keyword arguments are passed to `self.save_result()`. Note + that the names and values of keyword arguments are NOT saved as + results to the hdf5 file; they are only used to provide values + for the optional arguments of `self.save_result()`. + + Examples: + >>> run = Run('path/to/an/hdf5/file.h5') # doctest: +SKIP + >>> a = 5 + >>> b = 2.48 + >>> run.save_results('result', a, 'other_result', b, overwrite=False) # doctest: +SKIP + """ names = args[::2] values = args[1::2] for name, value in zip(names, values): From c4203f3741c972c0a3c4b96643d4cfc5e687bd44 Mon Sep 17 00:00:00 2001 From: Zak V Date: Tue, 3 Nov 2020 14:25:28 -0500 Subject: [PATCH 07/22] Removed print statement from Run.save_results(). --- lyse/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lyse/__init__.py b/lyse/__init__.py index b46d807..074155b 100644 --- a/lyse/__init__.py +++ b/lyse/__init__.py @@ -385,7 +385,6 @@ def save_results(self, *args, **kwargs): names = args[::2] values = args[1::2] for name, value in zip(names, values): - print('saving %s =' % name, value) self.save_result(name, value, **kwargs) def save_results_dict(self, results_dict, uncertainties=False, **kwargs): From d6c6919b74ec3ea8752b12682b0eba195fb9e41a Mon Sep 17 00:00:00 2001 From: Zak V Date: Tue, 3 Nov 2020 14:29:47 -0500 Subject: [PATCH 08/22] Small update to docstring for Run.save_result(). --- lyse/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lyse/__init__.py b/lyse/__init__.py index 074155b..ac772d1 100644 --- a/lyse/__init__.py +++ b/lyse/__init__.py @@ -257,8 +257,9 @@ def save_result(self, name, value, group=None, overwrite=True): result will be saved as an attribute. If set to `None`, then the result will be saved to `self.group` in `'/results'`. Note that if a value is passed for `group` here then it will NOT have - `'/result'` prepended to it. This is in contrast to using the - default group set with `self.set_group()`; when the default + `'/result'` prepended to it which allows the caller to save + results anywhere in the hdf5 file. This is in contrast to using + the default group set with `self.set_group()`; when the default group is set with that method it WILL have `'/results'` prepended to it when before saving results. Defaults to `None`. overwrite (bool, optional): Sets whether or not to overwrite the From 0bd4a020743ad967cad145ac4fc4a9e6193b5e87 Mon Sep 17 00:00:00 2001 From: Zak V Date: Tue, 3 Nov 2020 14:54:46 -0500 Subject: [PATCH 09/22] Changed Run's h5_path, no_write, and group attributes into properties. --- lyse/__init__.py | 54 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/lyse/__init__.py b/lyse/__init__.py index ac772d1..10eae3d 100644 --- a/lyse/__init__.py +++ b/lyse/__init__.py @@ -114,23 +114,11 @@ class Run(object): no_write (bool, optional): Set to `True` to prevent editing the run file. Note that doing so prohibits the ability to save results to the file. Defaults to `False`. - - Attributes: - h5_path (str): The value provided for `h5_path` during instantiation. - no_write (bool): The value provided for `no_write` during instantiation. - group (str): The group in the hdf5 file in which results are saved by - default. When a `Run` instance is created from within a lyse - singleshot or multishot routine, `group` will be set to the name of - the running routine. If created from outside a lyse script it will - be set to `None`. To change the default group for saving results, - use the `set_group()` method. Note that if `self.group` is `None` - and no value is provided for the optional `group` argument used by - the `save...()` methods, a `ValueError` will be raised. """ def __init__(self,h5_path,no_write=False): - self.no_write = no_write - self.group = None - self.h5_path = h5_path + self.__h5_path = h5_path + self.__no_write = no_write + self.__group = None if not self.no_write: self._create_group_if_not_exists(h5_path, '/', 'results') @@ -142,8 +130,7 @@ def __init__(self,h5_path,no_write=False): frame = inspect.currentframe() __file__ = frame.f_back.f_globals['__file__'] group = os.path.basename(__file__).split('.py')[0] - self._create_group_if_not_exists(h5_path, 'results', group) - self.group = group + self.set_group(group) except KeyError: # sys.stderr.write('Warning: to write results, call ' # 'Run.set_group(groupname), specifying the name of the group ' @@ -152,6 +139,37 @@ def __init__(self,h5_path,no_write=False): # 'mode, there is no script name.\n') pass + @property + def h5_path(self): + """str: The value provided for `h5_path` during instantiation.""" + return self.__h5_path + + @property + def no_write(self): + """bool: The value provided for `no_write` during instantiation.""" + return self.__no_write + + @property + def group(self): + """str: The group in the hdf5 file in which results are saved by default. + + When a `Run` instance is created from within a lyse singleshot or + multishot routine, `group` will be set to the name of the running + routine. If created from outside a lyse script it will be set to + `None`. To change the default group for saving results, use the + `set_group()` method. Note that if `self.group` is `None` and no + value is provided for the optional `group` argument used by the + `save...()` methods, a `ValueError` will be raised. + + Attempting to directly set `self.group`'s value will automatically + call `self.set_group()`. + """ + return self.__group + + @group.setter + def group(self, value): + self.set_group(value) + def _create_group_if_not_exists(self, h5_path, location, groupname): """Creates a group in the HDF5 file at `location` if it does not exist. @@ -188,7 +206,7 @@ def set_group(self, groupname): `'/results'` group of the hdf5 file. """ self._create_group_if_not_exists(self.h5_path, '/results', groupname) - self.group = groupname + self.__group = groupname def trace_names(self): with h5py.File(self.h5_path, 'r') as h5_file: From 575fb4fbb6303314da00167da78bde3f1e2f82e0 Mon Sep 17 00:00:00 2001 From: Zak V Date: Tue, 3 Nov 2020 14:56:32 -0500 Subject: [PATCH 10/22] Fixed minor typo in comment in Run class. --- lyse/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lyse/__init__.py b/lyse/__init__.py index 10eae3d..944c417 100644 --- a/lyse/__init__.py +++ b/lyse/__init__.py @@ -124,9 +124,9 @@ def __init__(self,h5_path,no_write=False): try: if not self.no_write: - # The group were this run's results will be stored in the h5 file - # will be the name of the python script which is instantiating - # this Run object: + # The group where this run's results will be stored in the h5 + # file will be the name of the python script which is + # instantiating this Run object: frame = inspect.currentframe() __file__ = frame.f_back.f_globals['__file__'] group = os.path.basename(__file__).split('.py')[0] From 60f00a48688bd6a3aa10d1971360e40921fb788d Mon Sep 17 00:00:00 2001 From: Zak V Date: Tue, 3 Nov 2020 15:04:28 -0500 Subject: [PATCH 11/22] Replaced "run" with "shot" in Run's docstring. --- lyse/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lyse/__init__.py b/lyse/__init__.py index 944c417..fd2b017 100644 --- a/lyse/__init__.py +++ b/lyse/__init__.py @@ -102,18 +102,18 @@ def globals_diff(run1, run2, group=None): return dict_diff(run1.get_globals(group), run2.get_globals(group)) class Run(object): - """A class for saving/retrieving data to/from a run's hdf5 file. + """A class for saving/retrieving data to/from a shot's hdf5 file. This class implements methods that allow the user to retrieve data from a - run's hdf5 file such as images, traces, and the values of globals. It also + shot's hdf5 file such as images, traces, and the values of globals. It also provides methods for saving and retrieving results from analysis. Args: h5_path (str): The path, including file name and extension, to the hdf5 - file for a run. - no_write (bool, optional): Set to `True` to prevent editing the run - file. Note that doing so prohibits the ability to save results to - the file. Defaults to `False`. + file for a shot. + no_write (bool, optional): Set to `True` to prevent editing the shot's + hdf5 file. Note that doing so prohibits the ability to save results + to the file. Defaults to `False`. """ def __init__(self,h5_path,no_write=False): self.__h5_path = h5_path From 9af179cd1236f8237179b3a3fab8bf6d14b578a0 Mon Sep 17 00:00:00 2001 From: Zak V Date: Tue, 3 Nov 2020 15:22:28 -0500 Subject: [PATCH 12/22] Cleaned up error messages in Run.save_result() and changed Exception to PermissionError to be more specific. --- lyse/__init__.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/lyse/__init__.py b/lyse/__init__.py index fd2b017..90026f3 100644 --- a/lyse/__init__.py +++ b/lyse/__init__.py @@ -286,20 +286,17 @@ def save_result(self, name, value, group=None, overwrite=True): raised. Defaults to `True`. Raises: - Exception: An `Exception` is raised if `self.no_write` is `True` - because saving the result would edit the file. + PermissionError: A `PermissionError` is raised if `self.no_write` is + `True` because saving the result would edit the file. ValueError: A `ValueError` is raised if `self.group` is `None` and no value is provided for `group` because the method then doesn't know where to save the result. - Exception: An `Exception` is raised if an attribute with name `name` - already exists but `overwrite` is set to `False`. + PermissionError: A `PermissionError` is raised if an attribute with + name `name` already exists but `overwrite` is set to `False`. """ if self.no_write: - raise Exception('This run is read-only. ' - 'You can\'t save results to runs through a ' - 'Sequence object. Per-run analysis should be done ' - 'in single-shot analysis routines, in which a ' - 'single Run object is used') + msg = "Cannot save result; this instance is read-only." + raise PermissionError(msg) with h5py.File(self.h5_path,'a') as h5_file: if not group: if self.group is None: @@ -314,8 +311,13 @@ def save_result(self, name, value, group=None, overwrite=True): # Create the group if it doesn't exist h5_file.create_group(group) if name in h5_file[group].attrs and not overwrite: - raise Exception('Attribute %s exists in group %s. ' \ - 'Use overwrite=True to overwrite.' % (name, group)) + msg = """Cannot save result; group '{group}' already has + attribute '{name}' and overwrite is set to False. Set + overwrite=True to overwrite the existing value.""".format( + group=group, + name=name, + ) + raise PermissionError(dedent(msg)) set_attributes(h5_file[group], {name: value}) if spinning_top: From cad1435a380ddf11131fc6ae316acb9e2a6d6996 Mon Sep 17 00:00:00 2001 From: Zak V Date: Tue, 3 Nov 2020 15:32:31 -0500 Subject: [PATCH 13/22] Updated error messages in Run.save_result_array() and replaced Exception with PermissionError to be more specific. --- lyse/__init__.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/lyse/__init__.py b/lyse/__init__.py index 90026f3..19d0131 100644 --- a/lyse/__init__.py +++ b/lyse/__init__.py @@ -333,11 +333,8 @@ def save_result_array(self, name, data, group=None, group in the 'results' group and overwrite existing data. Additional keyword arguments are passed directly to h5py.create_dataset().""" if self.no_write: - raise Exception('This run is read-only. ' - 'You can\'t save results to runs through a ' - 'Sequence object. Per-run analysis should be done ' - 'in single-shot analysis routines, in which a ' - 'single Run object is used') + msg = "Cannot save result; this instance is read-only." + raise PermissionError(msg) with h5py.File(self.h5_path, 'a') as h5_file: attrs = {} if not group: @@ -359,8 +356,14 @@ def save_result_array(self, name, data, group=None, attrs = dict(h5_file[group][name].attrs) del h5_file[group][name] else: - raise Exception('Dataset %s exists. Use overwrite=True to overwrite.' % - group + '/' + name) + msg = """Cannot save result; group '{group}' already has + dataset '{name}' and overwrite is set to False. Set + overwrite=True to overwrite the existing + value.""".format( + group=group, + name=name, + ) + raise PermissionError(dedent(msg)) h5_file[group].create_dataset(name, data=data, **kwargs) for key, val in attrs.items(): h5_file[group][name].attrs[key] = val From 9a590e566971d7d16d8167f95c86125dd0e16adf Mon Sep 17 00:00:00 2001 From: Zak V Date: Tue, 3 Nov 2020 15:32:58 -0500 Subject: [PATCH 14/22] Removed some trailing whitespace in lyse.Run(). --- lyse/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lyse/__init__.py b/lyse/__init__.py index 19d0131..04e0cc0 100644 --- a/lyse/__init__.py +++ b/lyse/__init__.py @@ -317,7 +317,7 @@ def save_result(self, name, value, group=None, overwrite=True): group=group, name=name, ) - raise PermissionError(dedent(msg)) + raise PermissionError(dedent(msg)) set_attributes(h5_file[group], {name: value}) if spinning_top: From e7778b543cf64088e3fdab61c1aba9e362276cb4 Mon Sep 17 00:00:00 2001 From: Zak V Date: Tue, 3 Nov 2020 15:49:17 -0500 Subject: [PATCH 15/22] Updated docstring for Run.save_result_array(). --- lyse/__init__.py | 40 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/lyse/__init__.py b/lyse/__init__.py index 04e0cc0..41d529c 100644 --- a/lyse/__init__.py +++ b/lyse/__init__.py @@ -329,9 +329,43 @@ def save_result(self, name, value, group=None, overwrite=True): def save_result_array(self, name, data, group=None, overwrite=True, keep_attrs=False, **kwargs): - """Save data array to h5 file. Defaults are to save to the active - group in the 'results' group and overwrite existing data. - Additional keyword arguments are passed directly to h5py.create_dataset().""" + """Save an array of data to the hdf5 h5 file. + + With the default argument values this method saves to `self.group` in + the `'/results'` group and overwrites any existing value without keeping + the dataset's previous attributes. Additional keyword arguments are + passed directly to `h5py.create_dataset()`. + + Args: + name (str): The name of the result. This will be the name of the + dataset added to the hdf5 file. + data (:obj:`numpy:numpy.array`): The data to save to the hdf5 file. + group (str, optional): The group in the hdf5 file in which the + result will be saved as a dataset. If set to `None`, then the + result will be saved in `self.group` in `'/results'`. Note that + if a value is passed for `group` here then it will NOT have + `'/result'` prepended to it which allows the caller to save + results anywhere in the hdf5 file. This is in contrast to using + the default group set with `self.set_group()`; when the default + group is set with that method it WILL have `'/results'` + prepended to it when saving results. Defaults to `None`.. + overwrite (bool, optional): Sets whether or not to overwrite the + previous value if the dataset already exists. If set to + `False` and the dataset already exists, a `PermissionError` is + raised. Defaults to `True`. + keep_attrs (bool, optional): Whether or not to keep the dataset's + attributes when overwriting it, i.e. if the dataset already + existed. Defaults to `False`. + + Raises: + PermissionError: A `PermissionError` is raised if `self.no_write` is + `True` because saving the result would edit the file. + ValueError: A `ValueError` is raised if `self.group` is `None` and + no value is provided for `group` because the method then doesn't + know where to save the result. + PermissionError: A `PermissionError` is raised if a dataset with + name `name` already exists but `overwrite` is set to `False`. + """ if self.no_write: msg = "Cannot save result; this instance is read-only." raise PermissionError(msg) From c6d8306c8b2a4f19d8a0493e686d851571b6c93c Mon Sep 17 00:00:00 2001 From: Zak V Date: Tue, 3 Nov 2020 15:49:50 -0500 Subject: [PATCH 16/22] Corrected typos in lyse.save_result()'s docstring. --- lyse/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lyse/__init__.py b/lyse/__init__.py index 41d529c..770632a 100644 --- a/lyse/__init__.py +++ b/lyse/__init__.py @@ -279,10 +279,10 @@ def save_result(self, name, value, group=None, overwrite=True): results anywhere in the hdf5 file. This is in contrast to using the default group set with `self.set_group()`; when the default group is set with that method it WILL have `'/results'` - prepended to it when before saving results. Defaults to `None`. + prepended to it when saving results. Defaults to `None`. overwrite (bool, optional): Sets whether or not to overwrite the previous value if the attribute already exists. If set to - `False` and the attribute already exists, an `Exception` is + `False` and the attribute already exists, a `PermissionError` is raised. Defaults to `True`. Raises: From 6a2755b5026fc7449f2cdf8388cd53dba06b5e68 Mon Sep 17 00:00:00 2001 From: Zak V Date: Tue, 3 Nov 2020 16:44:09 -0500 Subject: [PATCH 17/22] Replaced double underscores with single for private Run attributes so that child classes, e.g. Sequence, can use them. --- lyse/__init__.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lyse/__init__.py b/lyse/__init__.py index 770632a..85044e2 100644 --- a/lyse/__init__.py +++ b/lyse/__init__.py @@ -116,9 +116,9 @@ class Run(object): to the file. Defaults to `False`. """ def __init__(self,h5_path,no_write=False): - self.__h5_path = h5_path - self.__no_write = no_write - self.__group = None + self._h5_path = h5_path + self._no_write = no_write + self._group = None if not self.no_write: self._create_group_if_not_exists(h5_path, '/', 'results') @@ -142,12 +142,12 @@ def __init__(self,h5_path,no_write=False): @property def h5_path(self): """str: The value provided for `h5_path` during instantiation.""" - return self.__h5_path + return self._h5_path @property def no_write(self): """bool: The value provided for `no_write` during instantiation.""" - return self.__no_write + return self._no_write @property def group(self): @@ -164,7 +164,7 @@ def group(self): Attempting to directly set `self.group`'s value will automatically call `self.set_group()`. """ - return self.__group + return self._group @group.setter def group(self, value): @@ -206,7 +206,7 @@ def set_group(self, groupname): `'/results'` group of the hdf5 file. """ self._create_group_if_not_exists(self.h5_path, '/results', groupname) - self.__group = groupname + self._group = groupname def trace_names(self): with h5py.File(self.h5_path, 'r') as h5_file: From 005163ed870d6c1efdd4b3eb54d7410f4075aa11 Mon Sep 17 00:00:00 2001 From: Zak V Date: Tue, 3 Nov 2020 20:06:50 -0500 Subject: [PATCH 18/22] Fixed issues with Sequence.__init__(). First, it was updated to work with recent changes to its parent class. Also it used to error out if the hdf5 file didn't already exist, as first pointed out in PR 73, but that is now resolved.. --- lyse/__init__.py | 48 +++++++++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 19 deletions(-) diff --git a/lyse/__init__.py b/lyse/__init__.py index 85044e2..99c6ca2 100644 --- a/lyse/__init__.py +++ b/lyse/__init__.py @@ -578,32 +578,42 @@ def globals_diff(self, other_run, group=None): class Sequence(Run): def __init__(self, h5_path, run_paths, no_write=False): - if isinstance(run_paths, pandas.DataFrame): - run_paths = run_paths['filepath'] - self.h5_path = h5_path - self.no_write = no_write + # Ensure file exists without affecting its last modification time if it + # already exists. + try: + with h5py.File(h5_path, 'r') as f: + pass + except OSError: + if no_write: + msg = "Cannot create the hdf5 file; this instance is read-only." + raise PermissionError(msg) + else: + with h5py.File(h5_path, 'a') as f: + pass + self._h5_path = h5_path + self._no_write = no_write + self._group = None if not self.no_write: self._create_group_if_not_exists(h5_path, '/', 'results') - + + if isinstance(run_paths, pandas.DataFrame): + run_paths = run_paths['filepath'] self.runs = {path: Run(path,no_write=True) for path in run_paths} - - # The group were the results will be stored in the h5 file will - # be the name of the python script which is instantiating this - # Sequence object: + + # The group where the results will be stored in the h5 file will be the + # name of the python script which is instantiating this Sequence object: frame = inspect.currentframe() try: __file__ = frame.f_back.f_locals['__file__'] - self.group = os.path.basename(__file__).split('.py')[0] - if not self.no_write: - self._create_group_if_not_exists(h5_path, 'results', self.group) + group = os.path.basename(__file__).split('.py')[0] + self.set_group(group) except KeyError: - sys.stderr.write('Warning: to write results, call ' - 'Sequence.set_group(groupname), specifying the name of the group ' - 'you would like to save results to. This normally comes from ' - 'the filename of your script, but since you\'re in interactive ' - 'mode, there is no script name. Opening in read only mode for ' - 'the moment.\n') - self.no_write = True + # sys.stderr.write('Warning: to write results, call ' + # 'Sequence.set_group(groupname), specifying the name of the group ' + # 'you would like to save results to. This normally comes from ' + # 'the filename of your script, but since you\'re in interactive ' + # 'mode, there is no script name.\n') + pass def get_trace(self,*args): return {path:run.get_trace(*args) for path,run in self.runs.items()} From ab8e44fca555947f95cce80a15f71060fce1c201 Mon Sep 17 00:00:00 2001 From: Zak V Date: Tue, 3 Nov 2020 20:07:41 -0500 Subject: [PATCH 19/22] Corrected indentation in Run.group's docstring. --- lyse/__init__.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lyse/__init__.py b/lyse/__init__.py index 99c6ca2..26eff51 100644 --- a/lyse/__init__.py +++ b/lyse/__init__.py @@ -153,16 +153,16 @@ def no_write(self): def group(self): """str: The group in the hdf5 file in which results are saved by default. - When a `Run` instance is created from within a lyse singleshot or - multishot routine, `group` will be set to the name of the running - routine. If created from outside a lyse script it will be set to - `None`. To change the default group for saving results, use the - `set_group()` method. Note that if `self.group` is `None` and no - value is provided for the optional `group` argument used by the - `save...()` methods, a `ValueError` will be raised. - - Attempting to directly set `self.group`'s value will automatically - call `self.set_group()`. + When a `Run` instance is created from within a lyse singleshot or + multishot routine, `group` will be set to the name of the running + routine. If created from outside a lyse script it will be set to `None`. + To change the default group for saving results, use the `set_group()` + method. Note that if `self.group` is `None` and no value is provided for + the optional `group` argument used by the `save...()` methods, a + `ValueError` will be raised. + + Attempting to directly set `self.group`'s value will automatically call + `self.set_group()`. """ return self._group From 3d38b3bd0626eb2ec11cef3f0cb4abb983a02df0 Mon Sep 17 00:00:00 2001 From: Zak V Date: Wed, 4 Nov 2020 03:45:40 -0500 Subject: [PATCH 20/22] Refactored Sequence.__init__() to use Run.__init__() per Phil's idea in PR #80. --- lyse/__init__.py | 44 ++++++++++++++++++++------------------------ 1 file changed, 20 insertions(+), 24 deletions(-) diff --git a/lyse/__init__.py b/lyse/__init__.py index 26eff51..4753caa 100644 --- a/lyse/__init__.py +++ b/lyse/__init__.py @@ -126,10 +126,19 @@ def __init__(self,h5_path,no_write=False): if not self.no_write: # The group where this run's results will be stored in the h5 # file will be the name of the python script which is - # instantiating this Run object: - frame = inspect.currentframe() - __file__ = frame.f_back.f_globals['__file__'] - group = os.path.basename(__file__).split('.py')[0] + # instantiating this Run object. Iterate from innermost caller + # to outermost. The name of the script will be one frame in + # from analysis_subprocess.py. + group = None + inner_frame = inspect.currentframe() + inner_file_name = self._frame_to_file_name(inner_frame) + while group is None: + outer_frame = inner_frame.f_back + outer_file_name = self._frame_to_file_name(outer_frame) + if outer_file_name == 'analysis_subprocess': + group = inner_file_name + inner_frame = outer_frame + inner_file_name = outer_file_name self.set_group(group) except KeyError: # sys.stderr.write('Warning: to write results, call ' @@ -139,6 +148,11 @@ def __init__(self,h5_path,no_write=False): # 'mode, there is no script name.\n') pass + def _frame_to_file_name(self, frame): + file_path = frame.f_globals['__file__'] + file_name = os.path.basename(file_path).split('.py')[0] + return file_name + @property def h5_path(self): """str: The value provided for `h5_path` during instantiation.""" @@ -590,31 +604,13 @@ def __init__(self, h5_path, run_paths, no_write=False): else: with h5py.File(h5_path, 'a') as f: pass - self._h5_path = h5_path - self._no_write = no_write - self._group = None - if not self.no_write: - self._create_group_if_not_exists(h5_path, '/', 'results') + + super().__init__(h5_path, no_write=no_write) if isinstance(run_paths, pandas.DataFrame): run_paths = run_paths['filepath'] self.runs = {path: Run(path,no_write=True) for path in run_paths} - # The group where the results will be stored in the h5 file will be the - # name of the python script which is instantiating this Sequence object: - frame = inspect.currentframe() - try: - __file__ = frame.f_back.f_locals['__file__'] - group = os.path.basename(__file__).split('.py')[0] - self.set_group(group) - except KeyError: - # sys.stderr.write('Warning: to write results, call ' - # 'Sequence.set_group(groupname), specifying the name of the group ' - # 'you would like to save results to. This normally comes from ' - # 'the filename of your script, but since you\'re in interactive ' - # 'mode, there is no script name.\n') - pass - def get_trace(self,*args): return {path:run.get_trace(*args) for path,run in self.runs.items()} From 7ba70629022f06892edb9e20828c39b8ba204221 Mon Sep 17 00:00:00 2001 From: Zak V Date: Wed, 4 Nov 2020 03:51:30 -0500 Subject: [PATCH 21/22] Changed Run's private attributes from single underscore to double underscore. --- lyse/__init__.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lyse/__init__.py b/lyse/__init__.py index 4753caa..f153897 100644 --- a/lyse/__init__.py +++ b/lyse/__init__.py @@ -116,9 +116,9 @@ class Run(object): to the file. Defaults to `False`. """ def __init__(self,h5_path,no_write=False): - self._h5_path = h5_path - self._no_write = no_write - self._group = None + self.__h5_path = h5_path + self.__no_write = no_write + self.__group = None if not self.no_write: self._create_group_if_not_exists(h5_path, '/', 'results') @@ -156,12 +156,12 @@ def _frame_to_file_name(self, frame): @property def h5_path(self): """str: The value provided for `h5_path` during instantiation.""" - return self._h5_path + return self.__h5_path @property def no_write(self): """bool: The value provided for `no_write` during instantiation.""" - return self._no_write + return self.__no_write @property def group(self): @@ -178,7 +178,7 @@ def group(self): Attempting to directly set `self.group`'s value will automatically call `self.set_group()`. """ - return self._group + return self.__group @group.setter def group(self, value): @@ -220,7 +220,7 @@ def set_group(self, groupname): `'/results'` group of the hdf5 file. """ self._create_group_if_not_exists(self.h5_path, '/results', groupname) - self._group = groupname + self.__group = groupname def trace_names(self): with h5py.File(self.h5_path, 'r') as h5_file: From 1dac68f61cac826303960306ebf28b8b3db7e4cf Mon Sep 17 00:00:00 2001 From: Zak V Date: Wed, 4 Nov 2020 12:54:52 -0500 Subject: [PATCH 22/22] Run.__init__() now properly handles when a lyse script is called analysis_subprocess.py. --- lyse/__init__.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/lyse/__init__.py b/lyse/__init__.py index f153897..6fdb93d 100644 --- a/lyse/__init__.py +++ b/lyse/__init__.py @@ -129,15 +129,24 @@ def __init__(self,h5_path,no_write=False): # instantiating this Run object. Iterate from innermost caller # to outermost. The name of the script will be one frame in # from analysis_subprocess.py. + analysis_subprocess_path = os.path.join( + LYSE_DIR, + 'analysis_subprocess.py', + ) group = None inner_frame = inspect.currentframe() - inner_file_name = self._frame_to_file_name(inner_frame) + inner_path = self._frame_to_path(inner_frame) + inner_file_name = self._path_to_file_name(inner_path) while group is None: + # self._frame_to_path() will raise a KeyError if this loop + # reaches the outermost caller. outer_frame = inner_frame.f_back - outer_file_name = self._frame_to_file_name(outer_frame) - if outer_file_name == 'analysis_subprocess': + outer_path = self._frame_to_path(outer_frame) + outer_file_name = self._path_to_file_name(outer_path) + if outer_path == analysis_subprocess_path: group = inner_file_name inner_frame = outer_frame + inner_path = outer_path inner_file_name = outer_file_name self.set_group(group) except KeyError: @@ -148,9 +157,12 @@ def __init__(self,h5_path,no_write=False): # 'mode, there is no script name.\n') pass - def _frame_to_file_name(self, frame): - file_path = frame.f_globals['__file__'] - file_name = os.path.basename(file_path).split('.py')[0] + def _frame_to_path(self, frame): + path = frame.f_globals['__file__'] + return path + + def _path_to_file_name(self, path): + file_name = os.path.basename(path).split('.py')[0] return file_name @property