diff --git a/lyse/__main__.py b/lyse/__main__.py index 2d67df0..cd81c00 100644 --- a/lyse/__main__.py +++ b/lyse/__main__.py @@ -1602,7 +1602,7 @@ def add_files(self, filepaths, new_row_data, done=False): # Update the Qt model: for filepath in to_add: self.update_row(filepath, dataframe_already_updated=True) - + app.filebox.set_add_shots_progress(None, None, None) @inmain_decorator() def get_first_incomplete(self): @@ -2243,23 +2243,23 @@ def on_save_dataframe_triggered(self, choose_folder=True): for sequence in sequences: sequence_df = pandas.DataFrame(df[df['sequence'] == sequence], columns=df.columns).dropna(axis=1, how='all') labscript = sequence_df['labscript'].iloc[0] - filename = "dataframe_{}_{}.msg".format(sequence.to_pydatetime().strftime("%Y%m%dT%H%M%S"),labscript[:-3]) + filename = "dataframe_{}_{}.parquet".format(sequence.to_pydatetime().strftime("%Y%m%dT%H%M%S"),labscript[:-3]) if not choose_folder: save_path = os.path.dirname(sequence_df['filepath'].iloc[0]) sequence_df.infer_objects() for col in sequence_df.columns : if sequence_df[col].dtype == object: sequence_df[col] = pandas.to_numeric(sequence_df[col], errors='ignore') - sequence_df.to_msgpack(os.path.join(save_path, filename)) + sequence_df.to_parquet(os.path.join(save_path, filename)) else: error_dialog('Dataframe is empty') def on_load_dataframe_triggered(self): - default = os.path.join(self.exp_config.get('paths', 'experiment_shot_storage'), 'dataframe.msg') + default = os.path.join(self.exp_config.get('paths', 'experiment_shot_storage'), 'dataframe.parquet') file = QtWidgets.QFileDialog.getOpenFileName(self.ui, 'Select dataframe file to load', default, - "dataframe files (*.msg)") + "dataframe files (*.parquet)") if type(file) is tuple: file, _ = file if not file: @@ -2268,8 +2268,8 @@ def on_load_dataframe_triggered(self): # Convert to standard platform specific path, otherwise Qt likes # forward slashes: file = os.path.abspath(file) - df = pandas.read_msgpack(file).sort_values("run time").reset_index() - + df = pandas.read_parquet(file).sort_values("run time").reset_index() + # Check for changes in the shot files since the dataframe was exported def changed_since(filepath, time): if os.path.isfile(filepath): @@ -2279,9 +2279,8 @@ def changed_since(filepath, time): filepaths = df["filepath"].tolist() changetime_cache = os.path.getmtime(file) - need_updating = np.where(map(lambda x: changed_since(x, changetime_cache), filepaths))[0] + need_updating = np.where(list(map(lambda x: changed_since(x, changetime_cache), filepaths)))[0] need_updating = np.sort(need_updating)[::-1] # sort in descending order to not remove the wrong items with pop - # Reload the files where changes where made since exporting for index in need_updating: filepath = filepaths.pop(index)