From fa9eedb5676e018946c10917e0ba17e325f6d9b8 Mon Sep 17 00:00:00 2001 From: David Meyer Date: Fri, 3 Dec 2021 09:00:29 -0500 Subject: [PATCH 1/2] Replace deprecated msgpack with python's pickle for saving dataframes. This keeps machinery in place to read msgpack files if the installed pandas supports it, but it will always save new dataframes as pickles. --- lyse/__main__.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/lyse/__main__.py b/lyse/__main__.py index 2c0a434..56e5798 100644 --- a/lyse/__main__.py +++ b/lyse/__main__.py @@ -22,6 +22,7 @@ import time import traceback import queue +import warnings # 3rd party imports: splash.update_text('importing numpy') @@ -41,6 +42,7 @@ from labscript_utils.qtwidgets.headerview_with_widgets import HorizontalHeaderViewWithWidgets from labscript_utils.qtwidgets.outputbox import OutputBox import labscript_utils.shared_drive as shared_drive +from labscript_utils import dedent from lyse.dataframe_utilities import (concat_with_padding, get_dataframe_from_shot, @@ -2245,23 +2247,23 @@ def on_save_dataframe_triggered(self, choose_folder=True): for sequence in sequences: sequence_df = pandas.DataFrame(df[df['sequence'] == sequence], columns=df.columns).dropna(axis=1, how='all') labscript = sequence_df['labscript'].iloc[0] - filename = "dataframe_{}_{}.msg".format(sequence.to_pydatetime().strftime("%Y%m%dT%H%M%S"),labscript[:-3]) + filename = "dataframe_{}_{}.pkl".format(sequence.to_pydatetime().strftime("%Y%m%dT%H%M%S"),labscript[:-3]) if not choose_folder: save_path = os.path.dirname(sequence_df['filepath'].iloc[0]) sequence_df.infer_objects() for col in sequence_df.columns : if sequence_df[col].dtype == object: sequence_df[col] = pandas.to_numeric(sequence_df[col], errors='ignore') - sequence_df.to_msgpack(os.path.join(save_path, filename)) + sequence_df.to_pickle(os.path.join(save_path, filename)) else: error_dialog('Dataframe is empty') def on_load_dataframe_triggered(self): - default = os.path.join(self.exp_config.get('paths', 'experiment_shot_storage'), 'dataframe.msg') + default = os.path.join(self.exp_config.get('paths', 'experiment_shot_storage'), 'dataframe.pkl') file = QtWidgets.QFileDialog.getOpenFileName(self.ui, 'Select dataframe file to load', default, - "dataframe files (*.msg)") + "dataframe files (*.pkl *.msg)") if type(file) is tuple: file, _ = file if not file: @@ -2270,7 +2272,22 @@ def on_load_dataframe_triggered(self): # Convert to standard platform specific path, otherwise Qt likes # forward slashes: file = os.path.abspath(file) - df = pandas.read_msgpack(file).sort_values("run time").reset_index() + if file.endswith('.msg'): + # try to read msgpack in case using older pandas + try: + df = pandas.read_msgpack(file).sort_values("run time").reset_index() + # raise a deprecation warning if this succeeds + msg = """msgpack support is being dropped by pandas >= 1.0.0. + Please resave this dataframe to use the new format.""" + warnings.warn(dedent(msg),DeprecationWarning) + except AttributeError as err: + # using newer pandas that can't read msg + msg = """msgpack is no longer supported by pandas. + To read this dataframe, you must downgrade pandas to < 1.0.0. + You can then read this dataframe and resave it with the new format.""" + raise DeprecationWarning(dedent(msg)) from err + else: + df = pandas.read_pickle(file).sort_values("run time").reset_index() # Check for changes in the shot files since the dataframe was exported def changed_since(filepath, time): From 915cc1c501cd4c8b37767c369e183e50d6f79c05 Mon Sep 17 00:00:00 2001 From: David Meyer Date: Fri, 3 Dec 2021 09:01:04 -0500 Subject: [PATCH 2/2] Adding the extra bugfixes found in #91. --- lyse/__main__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lyse/__main__.py b/lyse/__main__.py index 56e5798..a104621 100644 --- a/lyse/__main__.py +++ b/lyse/__main__.py @@ -1606,6 +1606,8 @@ def add_files(self, filepaths, new_row_data, done=False): # Update the Qt model: for filepath in to_add: self.update_row(filepath, dataframe_already_updated=True) + + app.filebox.set_add_shots_progress(None, None, None) @inmain_decorator() @@ -2298,7 +2300,7 @@ def changed_since(filepath, time): filepaths = df["filepath"].tolist() changetime_cache = os.path.getmtime(file) - need_updating = np.where(map(lambda x: changed_since(x, changetime_cache), filepaths))[0] + need_updating = np.where(list(map(lambda x: changed_since(x, changetime_cache), filepaths)))[0] need_updating = np.sort(need_updating)[::-1] # sort in descending order to not remove the wrong items with pop # Reload the files where changes where made since exporting