diff --git a/lyse/__main__.py b/lyse/__main__.py index 2c0a434..a104621 100644 --- a/lyse/__main__.py +++ b/lyse/__main__.py @@ -22,6 +22,7 @@ import time import traceback import queue +import warnings # 3rd party imports: splash.update_text('importing numpy') @@ -41,6 +42,7 @@ from labscript_utils.qtwidgets.headerview_with_widgets import HorizontalHeaderViewWithWidgets from labscript_utils.qtwidgets.outputbox import OutputBox import labscript_utils.shared_drive as shared_drive +from labscript_utils import dedent from lyse.dataframe_utilities import (concat_with_padding, get_dataframe_from_shot, @@ -1604,6 +1606,8 @@ def add_files(self, filepaths, new_row_data, done=False): # Update the Qt model: for filepath in to_add: self.update_row(filepath, dataframe_already_updated=True) + + app.filebox.set_add_shots_progress(None, None, None) @inmain_decorator() @@ -2245,23 +2249,23 @@ def on_save_dataframe_triggered(self, choose_folder=True): for sequence in sequences: sequence_df = pandas.DataFrame(df[df['sequence'] == sequence], columns=df.columns).dropna(axis=1, how='all') labscript = sequence_df['labscript'].iloc[0] - filename = "dataframe_{}_{}.msg".format(sequence.to_pydatetime().strftime("%Y%m%dT%H%M%S"),labscript[:-3]) + filename = "dataframe_{}_{}.pkl".format(sequence.to_pydatetime().strftime("%Y%m%dT%H%M%S"),labscript[:-3]) if not choose_folder: save_path = os.path.dirname(sequence_df['filepath'].iloc[0]) sequence_df.infer_objects() for col in sequence_df.columns : if sequence_df[col].dtype == object: sequence_df[col] = pandas.to_numeric(sequence_df[col], errors='ignore') - sequence_df.to_msgpack(os.path.join(save_path, filename)) + sequence_df.to_pickle(os.path.join(save_path, filename)) else: error_dialog('Dataframe is empty') def on_load_dataframe_triggered(self): - default = os.path.join(self.exp_config.get('paths', 'experiment_shot_storage'), 'dataframe.msg') + default = os.path.join(self.exp_config.get('paths', 'experiment_shot_storage'), 'dataframe.pkl') file = QtWidgets.QFileDialog.getOpenFileName(self.ui, 'Select dataframe file to load', default, - "dataframe files (*.msg)") + "dataframe files (*.pkl *.msg)") if type(file) is tuple: file, _ = file if not file: @@ -2270,7 +2274,22 @@ def on_load_dataframe_triggered(self): # Convert to standard platform specific path, otherwise Qt likes # forward slashes: file = os.path.abspath(file) - df = pandas.read_msgpack(file).sort_values("run time").reset_index() + if file.endswith('.msg'): + # try to read msgpack in case using older pandas + try: + df = pandas.read_msgpack(file).sort_values("run time").reset_index() + # raise a deprecation warning if this succeeds + msg = """msgpack support is being dropped by pandas >= 1.0.0. + Please resave this dataframe to use the new format.""" + warnings.warn(dedent(msg),DeprecationWarning) + except AttributeError as err: + # using newer pandas that can't read msg + msg = """msgpack is no longer supported by pandas. + To read this dataframe, you must downgrade pandas to < 1.0.0. + You can then read this dataframe and resave it with the new format.""" + raise DeprecationWarning(dedent(msg)) from err + else: + df = pandas.read_pickle(file).sort_values("run time").reset_index() # Check for changes in the shot files since the dataframe was exported def changed_since(filepath, time): @@ -2281,7 +2300,7 @@ def changed_since(filepath, time): filepaths = df["filepath"].tolist() changetime_cache = os.path.getmtime(file) - need_updating = np.where(map(lambda x: changed_since(x, changetime_cache), filepaths))[0] + need_updating = np.where(list(map(lambda x: changed_since(x, changetime_cache), filepaths)))[0] need_updating = np.sort(need_updating)[::-1] # sort in descending order to not remove the wrong items with pop # Reload the files where changes where made since exporting