From 2447f67ab2c90f12ff6072f633b7bf48f2bc5dab Mon Sep 17 00:00:00 2001 From: Lars Pause Date: Wed, 24 Feb 2021 14:20:51 +0100 Subject: [PATCH 1/4] Use to_parquet to save dataframe As pandas.to_msgpack is removed since pandas 1.x.x, there is an alternative needed. For to-feather I get problems due to the multiindex structure of the dataframes. to_parquet does not have this problem if pandas >= version 1.2 (https://stackoverflow.com/questions/54861430/how-do-i-save-multi-indexed-pandas-dataframes-to-parquet) --- lyse/__main__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lyse/__main__.py b/lyse/__main__.py index 2d67df0..986b468 100644 --- a/lyse/__main__.py +++ b/lyse/__main__.py @@ -2243,23 +2243,23 @@ def on_save_dataframe_triggered(self, choose_folder=True): for sequence in sequences: sequence_df = pandas.DataFrame(df[df['sequence'] == sequence], columns=df.columns).dropna(axis=1, how='all') labscript = sequence_df['labscript'].iloc[0] - filename = "dataframe_{}_{}.msg".format(sequence.to_pydatetime().strftime("%Y%m%dT%H%M%S"),labscript[:-3]) + filename = "dataframe_{}_{}.ftr".format(sequence.to_pydatetime().strftime("%Y%m%dT%H%M%S"),labscript[:-3]) if not choose_folder: save_path = os.path.dirname(sequence_df['filepath'].iloc[0]) sequence_df.infer_objects() for col in sequence_df.columns : if sequence_df[col].dtype == object: sequence_df[col] = pandas.to_numeric(sequence_df[col], errors='ignore') - sequence_df.to_msgpack(os.path.join(save_path, filename)) + sequence_df.to_parquet(os.path.join(save_path, filename)) else: error_dialog('Dataframe is empty') def on_load_dataframe_triggered(self): - default = os.path.join(self.exp_config.get('paths', 'experiment_shot_storage'), 'dataframe.msg') + default = os.path.join(self.exp_config.get('paths', 'experiment_shot_storage'), 'dataframe.ftr') file = QtWidgets.QFileDialog.getOpenFileName(self.ui, 'Select dataframe file to load', default, - "dataframe files (*.msg)") + "dataframe files (*.ftr)") if type(file) is tuple: file, _ = file if not file: @@ -2268,8 +2268,8 @@ def on_load_dataframe_triggered(self): # Convert to standard platform specific path, otherwise Qt likes # forward slashes: file = os.path.abspath(file) - df = pandas.read_msgpack(file).sort_values("run time").reset_index() - + df = pandas.read_parquet(file).sort_values("run time").reset_index() + # Check for changes in the shot files since the dataframe was exported def changed_since(filepath, time): if os.path.isfile(filepath): From 8bbee7f33ed64ab5f6363495789e585238966a35 Mon Sep 17 00:00:00 2001 From: Lars Kohfahl Date: Thu, 25 Feb 2021 14:28:49 +0100 Subject: [PATCH 2/4] rename file ending .ftr to .parquet As we do not use feather, the file ending .parquet is better --- lyse/__main__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lyse/__main__.py b/lyse/__main__.py index 986b468..e54f489 100644 --- a/lyse/__main__.py +++ b/lyse/__main__.py @@ -2243,7 +2243,7 @@ def on_save_dataframe_triggered(self, choose_folder=True): for sequence in sequences: sequence_df = pandas.DataFrame(df[df['sequence'] == sequence], columns=df.columns).dropna(axis=1, how='all') labscript = sequence_df['labscript'].iloc[0] - filename = "dataframe_{}_{}.ftr".format(sequence.to_pydatetime().strftime("%Y%m%dT%H%M%S"),labscript[:-3]) + filename = "dataframe_{}_{}.parquet".format(sequence.to_pydatetime().strftime("%Y%m%dT%H%M%S"),labscript[:-3]) if not choose_folder: save_path = os.path.dirname(sequence_df['filepath'].iloc[0]) sequence_df.infer_objects() @@ -2255,11 +2255,11 @@ def on_save_dataframe_triggered(self, choose_folder=True): error_dialog('Dataframe is empty') def on_load_dataframe_triggered(self): - default = os.path.join(self.exp_config.get('paths', 'experiment_shot_storage'), 'dataframe.ftr') + default = os.path.join(self.exp_config.get('paths', 'experiment_shot_storage'), 'dataframe.parquet') file = QtWidgets.QFileDialog.getOpenFileName(self.ui, 'Select dataframe file to load', default, - "dataframe files (*.ftr)") + "dataframe files (*.parquet)") if type(file) is tuple: file, _ = file if not file: From 39121a73632f6237af0306ff5f05916faa9cdb7f Mon Sep 17 00:00:00 2001 From: Lars Kohfahl Date: Tue, 16 Mar 2021 16:21:23 +0100 Subject: [PATCH 3/4] bugfix load_dataframe When loading a dataframe, the first shot was always detected as "needs updating" in on_load_dataframe_triggered(). This was because np.where did not work correctly on the map-object. Converting the map-object first to a list resolves the problem. --- lyse/__main__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lyse/__main__.py b/lyse/__main__.py index e54f489..82b5953 100644 --- a/lyse/__main__.py +++ b/lyse/__main__.py @@ -2279,9 +2279,8 @@ def changed_since(filepath, time): filepaths = df["filepath"].tolist() changetime_cache = os.path.getmtime(file) - need_updating = np.where(map(lambda x: changed_since(x, changetime_cache), filepaths))[0] + need_updating = np.where(list(map(lambda x: changed_since(x, changetime_cache), filepaths)))[0] need_updating = np.sort(need_updating)[::-1] # sort in descending order to not remove the wrong items with pop - # Reload the files where changes where made since exporting for index in need_updating: filepath = filepaths.pop(index) From b252312713792a6d75961a2c9ec78c77e265346b Mon Sep 17 00:00:00 2001 From: Lars Pause Date: Tue, 16 Mar 2021 16:43:10 +0100 Subject: [PATCH 4/4] bugfix: Hide add_shots_progress-bar when loading dataframe --- lyse/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lyse/__main__.py b/lyse/__main__.py index 82b5953..cd81c00 100644 --- a/lyse/__main__.py +++ b/lyse/__main__.py @@ -1602,7 +1602,7 @@ def add_files(self, filepaths, new_row_data, done=False): # Update the Qt model: for filepath in to_add: self.update_row(filepath, dataframe_already_updated=True) - + app.filebox.set_add_shots_progress(None, None, None) @inmain_decorator() def get_first_incomplete(self):