From c75b9451ba9e62216880312d74d8496daea78d90 Mon Sep 17 00:00:00 2001
From: Zak V <zakven@mit.edu>
Date: Wed, 28 Oct 2020 19:01:36 -0400
Subject: [PATCH 01/15] Added optional n_sequences argument to lyse.data().

---
 lyse/__init__.py | 13 +++++++++++--
 lyse/__main__.py | 47 +++++++++++++++++++++++++++++++++++++++++++++--
 2 files changed, 56 insertions(+), 4 deletions(-)

diff --git a/lyse/__init__.py b/lyse/__init__.py
index 6163b8a..66ab133 100644
--- a/lyse/__init__.py
+++ b/lyse/__init__.py
@@ -76,11 +76,20 @@ class _RoutineStorage(object):
 routine_storage = _RoutineStorage()
 
 
-def data(filepath=None, host='localhost', port=_lyse_port, timeout=5):
+def data(filepath=None, host='localhost', port=_lyse_port, timeout=5, n_sequences=None):
     if filepath is not None:
         return _get_singleshot(filepath)
     else:
-        df = zmq_get(port, host, 'get dataframe', timeout)
+        command = 'get dataframe'
+        if n_sequences:
+            if type(n_sequences) is int and n_sequences > 0:
+                command = command + ' n_sequences={n_sequences}'.format(n_sequences=n_sequences)
+            else:
+                msg = """n_sequences must be None or an integer greater than 0 but 
+                    was {n_sequences}.""".format(n_sequences=n_sequences)
+                raise ValueError(dedent(msg))
+        df = zmq_get(port, host, command, timeout)
+
         try:
             padding = ('',)*(df.columns.nlevels - 1)
             try:
diff --git a/lyse/__main__.py b/lyse/__main__.py
index 0d9a943..f5d7bfe 100644
--- a/lyse/__main__.py
+++ b/lyse/__main__.py
@@ -156,14 +156,20 @@ def handler(self, request_data):
         logger.info('WebServer request: %s' % str(request_data))
         if request_data == 'hello':
             return 'hello'
-        elif request_data == 'get dataframe':
+        elif type(request_data) is str and request_data.startswith('get dataframe'):
             # infer_objects() picks fixed datatypes for columns that are compatible with
             # fixed datatypes, dramatically speeding up pickling. It is called here
             # rather than when updating the dataframe as calling it during updating may
             # call it needlessly often, whereas it only needs to be called prior to
             # sending the dataframe to a client requesting it, as we're doing now.
             app.filebox.shots_model.infer_objects()
-            return app.filebox.shots_model.dataframe
+            df = app.filebox.shots_model.dataframe
+            # Return only a subset of the dataframe if instructed to do so.
+            arguments = request_data.replace('get dataframe', '').strip()
+            if arguments.startswith('n_sequences='):
+                n_sequences = int(arguments.replace('n_sequences=', ''))
+                df = self._extract_n_sequences_from_df(df, n_sequences)
+            return df
         elif isinstance(request_data, dict):
             if 'filepath' in request_data:
                 h5_filepath = shared_drive.path_to_local(request_data['filepath'])
@@ -181,6 +187,43 @@ def handler(self, request_data):
         return ("error: operation not supported. Recognised requests are:\n "
                 "'get dataframe'\n 'hello'\n {'filepath': <some_h5_filepath>}")
 
+    def _extract_n_sequences_from_df(self, df, n_sequences):
+        # If the dataframe is empty, just return it, otherwise accessing columns
+        # below will raise a KeyError.
+        if df.empty:
+            return df
+
+        # Get a list of all unique sequences, each corresponding to one call to
+        # engage in runmanager. Each sequence may contain multiple runs. The
+        # below creates strings to identify sequences. To be from the same
+        # sequence, two shots have to have the same value for 'sequence' (which
+        # makes sure that the time when engage was called are the same to within
+        # 1 second), 'labscript' (must have been generated from the same
+        # labscript), and 'sequence_index' (a counter which keeps track of how
+        # many times engage has been called and resets to 0 at the start of each
+        # day). Typically just the value for sequence, is enough. However it
+        # only records time down to the second, so if engage() is called twice
+        # quickly then two different sequences can end up with the same value
+        # there.
+        sequences = [str(sequence) for sequence in df['sequence']]
+        labscripts = [str(labscript) for labscript in df['labscript']]
+        sequence_indices = [str(index) for index in df['sequence_index']]
+        # Combine into one string.
+        criteria = zip(sequences, labscripts, sequence_indices)
+        indentity_strings = [seq + script + ind for seq, script, ind in criteria]
+
+        # Find the distinct values, maintaining their ordering.
+        unique_identities = np.intersect1d(indentity_strings, indentity_strings)
+
+        # Slice the DataFrame so that only the last n_sequences sequences
+        # remain. Note that slicing unique_identities just returns all of its
+        # entries if n_sequences is greater than its length; it doesn't raise an
+        # error.
+        identities_included = unique_identities[-n_sequences:]
+        df_subset = df[[id in identities_included for id in indentity_strings]]
+
+        return df_subset
+
 
 class LyseMainWindow(QtWidgets.QMainWindow):
     # A signal to show that the window is shown and painted.

From 9d662eac920c744df1bb225c3592653b826b6f03 Mon Sep 17 00:00:00 2001
From: Zak V <zakven@mit.edu>
Date: Wed, 28 Oct 2020 20:00:43 -0400
Subject: [PATCH 02/15] lyse.data() now correctly handles when n_sequences=0.

---
 lyse/__init__.py | 4 ++--
 lyse/__main__.py | 5 ++++-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/lyse/__init__.py b/lyse/__init__.py
index 66ab133..9ed4894 100644
--- a/lyse/__init__.py
+++ b/lyse/__init__.py
@@ -81,8 +81,8 @@ def data(filepath=None, host='localhost', port=_lyse_port, timeout=5, n_sequence
         return _get_singleshot(filepath)
     else:
         command = 'get dataframe'
-        if n_sequences:
-            if type(n_sequences) is int and n_sequences > 0:
+        if n_sequences is not None:
+            if type(n_sequences) is int and n_sequences >= 0:
                 command = command + ' n_sequences={n_sequences}'.format(n_sequences=n_sequences)
             else:
                 msg = """n_sequences must be None or an integer greater than 0 but 
diff --git a/lyse/__main__.py b/lyse/__main__.py
index f5d7bfe..adcf200 100644
--- a/lyse/__main__.py
+++ b/lyse/__main__.py
@@ -219,7 +219,10 @@ def _extract_n_sequences_from_df(self, df, n_sequences):
         # remain. Note that slicing unique_identities just returns all of its
         # entries if n_sequences is greater than its length; it doesn't raise an
         # error.
-        identities_included = unique_identities[-n_sequences:]
+        if n_sequences == 0:
+            identities_included = []
+        else:
+            identities_included = unique_identities[-n_sequences:]
         df_subset = df[[id in identities_included for id in indentity_strings]]
 
         return df_subset

From c25caef3fff8dbd33078676a8eb396d7cb65c9e9 Mon Sep 17 00:00:00 2001
From: Zak V <zakven@mit.edu>
Date: Wed, 28 Oct 2020 20:36:02 -0400
Subject: [PATCH 03/15] Added a docstring for lyse.data().

---
 lyse/__init__.py | 42 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/lyse/__init__.py b/lyse/__init__.py
index 9ed4894..53d6994 100644
--- a/lyse/__init__.py
+++ b/lyse/__init__.py
@@ -77,6 +77,48 @@ class _RoutineStorage(object):
 
 
 def data(filepath=None, host='localhost', port=_lyse_port, timeout=5, n_sequences=None):
+    """Get data from the lyse dataframe or a file.
+    
+    This function allows for either extracting information from a run's hdf5
+    file, or retrieving data from lyse's dataframe. If `filepath` is provided
+    then data will be read from that file and returned as a pandas series. If
+    `filepath` is not provided then the dataframe in lyse, or a portion of it,
+    will be returned.
+
+    Args:
+        filepath (str, optional): The path to a run's hdf5 file. If a value
+            other than `None` is provided, then this function will return a
+            pandas series containing data associated with the run. In particular
+            it will contain the globals, singleshot results, multishot results,
+            etc. that would appear in the run's row in the Lyse dataframe, but
+            the values will be read from the file rather than extracted from the
+            lyse dataframe. If `filepath` is `None, then this function will
+            instead return a section of the lyse dataframe. Note that if
+            `filepath` is not None, then the other arguments will be ignored.
+            Defaults to `None`.
+        host (str, optional): The address of the computer running lyse. Defaults
+            to `'localhost'`.
+        port (int, optional): The port on which lyse is listening. Defaults to
+            the entry for lyse's port in the labconfig, with a fallback value of
+            42519 if the labconfig has no such entry.
+        timeout (float, optional): The timeout, in seconds, for the
+            communication with lyse. Defaults to 5.
+        n_sequences (int, optional): The number of sequences to include in the
+            returned dataframe where one sequence corresponds to one call to
+            engage in runmanager. The dataframe rows for the most recent
+            `n_sequences` sequences are returned. If set to `None`, then all
+            rows are returned. Defaults to `None`.
+
+    Raises:
+        ValueError: If `n_sequences` isn't `None` or a nonnegative integer, then
+            a `ValueError` is raised.
+
+    Returns:
+        (pandas series or dataframe): If `filepath` is provided, then a pandas
+            series with the data read from that file is returned. If `filepath`
+            is omitted or set to `None` then the lyse dataframe, or a subset of
+            it, is returned.
+    """    
     if filepath is not None:
         return _get_singleshot(filepath)
     else:

From d1d9897c987d0e2d45991e9ac335db19dd527a66 Mon Sep 17 00:00:00 2001
From: Zak V <zakven@mit.edu>
Date: Wed, 28 Oct 2020 21:42:03 -0400
Subject: [PATCH 04/15] Clarified lyse.data()'s docstring about behavior when
 n_sequences is greater than number of sequences available.

---
 lyse/__init__.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/lyse/__init__.py b/lyse/__init__.py
index 53d6994..206885b 100644
--- a/lyse/__init__.py
+++ b/lyse/__init__.py
@@ -103,15 +103,19 @@ def data(filepath=None, host='localhost', port=_lyse_port, timeout=5, n_sequence
             42519 if the labconfig has no such entry.
         timeout (float, optional): The timeout, in seconds, for the
             communication with lyse. Defaults to 5.
-        n_sequences (int, optional): The number of sequences to include in the
-            returned dataframe where one sequence corresponds to one call to
-            engage in runmanager. The dataframe rows for the most recent
-            `n_sequences` sequences are returned. If set to `None`, then all
-            rows are returned. Defaults to `None`.
+        n_sequences (int, optional): The maximum number of sequences to include
+            in the returned dataframe where one sequence corresponds to one call
+            to engage in runmanager. The dataframe rows for the most recent
+            `n_sequences` sequences are returned. If the dataframe contains
+            fewer than `n_sequences` sequences, then all rows will be returned.
+            If set to `None`, then all rows are returned. Defaults to `None`.
 
     Raises:
         ValueError: If `n_sequences` isn't `None` or a nonnegative integer, then
-            a `ValueError` is raised.
+            a `ValueError` is raised. Note that no `ValueError` is raised if
+            `n_sequences` is greater than the number of sequences available. In
+            that case as all available sequences are returned, i.e. the entire
+            lyse dataframe is returned.
 
     Returns:
         (pandas series or dataframe): If `filepath` is provided, then a pandas

From d134b6ecf6a2c5f7d9ddf985145f5fbac04cfd31 Mon Sep 17 00:00:00 2001
From: Zak V <zakven@mit.edu>
Date: Thu, 29 Oct 2020 12:03:23 -0400
Subject: [PATCH 05/15] Improved WebServer's thread safety.

---
 lyse/__main__.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/lyse/__main__.py b/lyse/__main__.py
index adcf200..fc99eb5 100644
--- a/lyse/__main__.py
+++ b/lyse/__main__.py
@@ -152,6 +152,7 @@ def get_screen_geometry():
 
 class WebServer(ZMQServer):
 
+    @inmain_decorator(wait_for_return=True)
     def handler(self, request_data):
         logger.info('WebServer request: %s' % str(request_data))
         if request_data == 'hello':
@@ -169,7 +170,15 @@ def handler(self, request_data):
             if arguments.startswith('n_sequences='):
                 n_sequences = int(arguments.replace('n_sequences=', ''))
                 df = self._extract_n_sequences_from_df(df, n_sequences)
-            return df
+            
+            # Returning the dataframe would mean that another thread would
+            # pickle/send it, which could happen partway through changes to the
+            # dataframe in the main thread. Sending it here prevents that thanks
+            # to the inmain_decorator (assuming that all modifications to the
+            # dataframe happen in the main thread). Then return NO_RESPONSE so
+            # that no additional message is sent.
+            self.send(df)
+            return self.NO_RESPONSE
         elif isinstance(request_data, dict):
             if 'filepath' in request_data:
                 h5_filepath = shared_drive.path_to_local(request_data['filepath'])
@@ -184,9 +193,6 @@ def handler(self, request_data):
             app.filebox.incoming_queue.put(shared_drive.path_to_local(request_data))
             return "Experiment added successfully\n"
 
-        return ("error: operation not supported. Recognised requests are:\n "
-                "'get dataframe'\n 'hello'\n {'filepath': <some_h5_filepath>}")
-
     def _extract_n_sequences_from_df(self, df, n_sequences):
         # If the dataframe is empty, just return it, otherwise accessing columns
         # below will raise a KeyError.

From f1be302dafdac9ef1736f62164630ffd3224ba53 Mon Sep 17 00:00:00 2001
From: Zak V <zakven@mit.edu>
Date: Thu, 29 Oct 2020 19:14:43 -0400
Subject: [PATCH 06/15] Added optional filter_kwargs argument to lyse.data().

---
 lyse/__init__.py | 15 ++++++++++++---
 lyse/__main__.py | 28 ++++++++++++++++++++++++----
 2 files changed, 36 insertions(+), 7 deletions(-)

diff --git a/lyse/__init__.py b/lyse/__init__.py
index 206885b..ceb55f8 100644
--- a/lyse/__init__.py
+++ b/lyse/__init__.py
@@ -19,6 +19,7 @@
 import inspect
 import sys
 import threading
+import shlex
 
 import labscript_utils.h5_lock, h5py
 from labscript_utils.labconfig import LabConfig
@@ -76,7 +77,7 @@ class _RoutineStorage(object):
 routine_storage = _RoutineStorage()
 
 
-def data(filepath=None, host='localhost', port=_lyse_port, timeout=5, n_sequences=None):
+def data(filepath=None, host='localhost', port=_lyse_port, timeout=5, n_sequences=None, filter_kwargs=None):
     """Get data from the lyse dataframe or a file.
     
     This function allows for either extracting information from a run's hdf5
@@ -126,14 +127,22 @@ def data(filepath=None, host='localhost', port=_lyse_port, timeout=5, n_sequence
     if filepath is not None:
         return _get_singleshot(filepath)
     else:
-        command = 'get dataframe'
+        command_list = ['get_dataframe']
         if n_sequences is not None:
             if type(n_sequences) is int and n_sequences >= 0:
-                command = command + ' n_sequences={n_sequences}'.format(n_sequences=n_sequences)
+                command_list.append('--n_sequences {n_sequences}'.format(n_sequences=n_sequences))
             else:
                 msg = """n_sequences must be None or an integer greater than 0 but 
                     was {n_sequences}.""".format(n_sequences=n_sequences)
                 raise ValueError(dedent(msg))
+        if filter_kwargs is not None:
+            if type(filter_kwargs) is dict:
+                command_list.append('--filter_kwargs ' + shlex.quote(repr(filter_kwargs)))
+            else:
+                msg = """filter must be None or a dictionary but was 
+                    {filter_kwargs}.""".format(filter_kwargs=filter_kwargs)
+                raise ValueError(dedent(msg))
+        command = ' '.join(command_list)
         df = zmq_get(port, host, command, timeout)
 
         try:
diff --git a/lyse/__main__.py b/lyse/__main__.py
index fc99eb5..4c6e90d 100644
--- a/lyse/__main__.py
+++ b/lyse/__main__.py
@@ -20,6 +20,9 @@
 import time
 import traceback
 import queue
+import argparse
+import shlex
+import ast
 
 # 3rd party imports:
 splash.update_text('importing numpy')
@@ -152,12 +155,28 @@ def get_screen_geometry():
 
 class WebServer(ZMQServer):
 
+    def __init__(self, *args, **kwargs):
+        ZMQServer.__init__(self, *args, **kwargs)
+        # Add parser for interpretting options for get_dataframe
+        parser = argparse.ArgumentParser()
+        parser.add_argument('--n_sequences', type=int)
+        parser.add_argument('--filter_kwargs', type=str)
+        self._command_parser = parser
+
     @inmain_decorator(wait_for_return=True)
     def handler(self, request_data):
         logger.info('WebServer request: %s' % str(request_data))
         if request_data == 'hello':
             return 'hello'
-        elif type(request_data) is str and request_data.startswith('get dataframe'):
+        elif type(request_data) is str and request_data.startswith('get_dataframe'):
+            # Parse any arguments.
+            command_args = shlex.split(request_data)[1:]  # Strip leading 'get_dataframe'.
+            parsed_args = self._command_parser.parse_args(command_args)
+            n_sequences = parsed_args.n_sequences
+            filter_kwargs = parsed_args.filter_kwargs
+            if filter_kwargs is not None:
+                filter_kwargs = ast.literal_eval(filter_kwargs)
+
             # infer_objects() picks fixed datatypes for columns that are compatible with
             # fixed datatypes, dramatically speeding up pickling. It is called here
             # rather than when updating the dataframe as calling it during updating may
@@ -165,11 +184,12 @@ def handler(self, request_data):
             # sending the dataframe to a client requesting it, as we're doing now.
             app.filebox.shots_model.infer_objects()
             df = app.filebox.shots_model.dataframe
+
             # Return only a subset of the dataframe if instructed to do so.
-            arguments = request_data.replace('get dataframe', '').strip()
-            if arguments.startswith('n_sequences='):
-                n_sequences = int(arguments.replace('n_sequences=', ''))
+            if n_sequences is not None:
                 df = self._extract_n_sequences_from_df(df, n_sequences)
+            if filter_kwargs is not None:
+                df = df.filter(**filter_kwargs)
             
             # Returning the dataframe would mean that another thread would
             # pickle/send it, which could happen partway through changes to the

From b61ab2522d4c764353e207d6c4087bf104417738 Mon Sep 17 00:00:00 2001
From: Zak V <zakven@mit.edu>
Date: Thu, 29 Oct 2020 19:30:50 -0400
Subject: [PATCH 07/15] Updated docstring for lyse.data().

---
 lyse/__init__.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/lyse/__init__.py b/lyse/__init__.py
index ceb55f8..6ac884c 100644
--- a/lyse/__init__.py
+++ b/lyse/__init__.py
@@ -85,6 +85,17 @@ def data(filepath=None, host='localhost', port=_lyse_port, timeout=5, n_sequence
     then data will be read from that file and returned as a pandas series. If
     `filepath` is not provided then the dataframe in lyse, or a portion of it,
     will be returned.
+    
+    Often only part of the lyse dataframe is needed, so the `n_sequences` and
+    `filter_kwargs` arguments provide ways to restrict what parts of the lyse
+    dataframe are returned. The dataframe can be quite large, so only requesting
+    a small part of it can speed up the execution of `lyse.data()` noticeably.
+    Setting `n_sequences` makes this function return only the rows of the lyse
+    dataframe that correspond to the `n_sequences` most recent sequences, where
+    one sequence corresponds to one call to engage in runmanager. Additionally,
+    the `Dataframe.filter()` method can be called on the dataframe before it is
+    transmitted, and the arguments specified in `filter_kwargs` are passed to
+    that method.
 
     Args:
         filepath (str, optional): The path to a run's hdf5 file. If a value
@@ -110,6 +121,13 @@ def data(filepath=None, host='localhost', port=_lyse_port, timeout=5, n_sequence
             `n_sequences` sequences are returned. If the dataframe contains
             fewer than `n_sequences` sequences, then all rows will be returned.
             If set to `None`, then all rows are returned. Defaults to `None`.
+        filter_kwargs (dict, optional): A dictionary of keyword arguments to
+            pass to the `Dataframe.filter()` method before the lyse dataframe is
+            returned. For example to call `filter()` with `like='temperature'`,
+            set `filter_kwargs` to `{'like':'temperature'}`. If set to `None`
+            then `Dataframe.filter()` will not be called. See
+            `Dataframe.filter()`'s documentation for more information. Defaults
+            to `None`.
 
     Raises:
         ValueError: If `n_sequences` isn't `None` or a nonnegative integer, then

From 8c7a456e30bd065dec165752313a2be6856a94a2 Mon Sep 17 00:00:00 2001
From: Zak V <zakven@mit.edu>
Date: Thu, 29 Oct 2020 22:21:57 -0400
Subject: [PATCH 08/15] Fixed an issue where the conversion to multiindex in
 lyse.data() could fail when using its filter_kwargs argument.

---
 lyse/__init__.py | 45 +++++++++++++++++++++++++++++----------------
 lyse/__main__.py |  3 ++-
 2 files changed, 31 insertions(+), 17 deletions(-)

diff --git a/lyse/__init__.py b/lyse/__init__.py
index 6ac884c..3abf4fe 100644
--- a/lyse/__init__.py
+++ b/lyse/__init__.py
@@ -162,24 +162,37 @@ def data(filepath=None, host='localhost', port=_lyse_port, timeout=5, n_sequence
                 raise ValueError(dedent(msg))
         command = ' '.join(command_list)
         df = zmq_get(port, host, command, timeout)
-
-        try:
-            padding = ('',)*(df.columns.nlevels - 1)
-            try:
-                integer_indexing = _labconfig.getboolean('lyse', 'integer_indexing')
-            except (LabConfig.NoOptionError, LabConfig.NoSectionError):
-                integer_indexing = False
-            if integer_indexing:
-                df.set_index(['sequence_index', 'run number', 'run repeat'], inplace=True, drop=False)
-            else:
-                df.set_index([('sequence',) + padding,('run time',) + padding], inplace=True, drop=False)
-                df.index.names = ['sequence', 'run time']
-        except KeyError:
-            # Empty DataFrame or index column not found, so fall back to RangeIndex instead
-            pass
+        # Ensure conversion to multiindex is done, which needs to be done here
+        # if the server is running an old version of lyse.
+        _rangeindex_to_multiindex(df, inplace=True)
         df.sort_index(inplace=True)
         return df
-        
+
+def _rangeindex_to_multiindex(df, inplace):
+    if isinstance(df.index, pandas.MultiIndex):
+        # The dataframe has already been converted.
+        return df
+    try:
+        padding = ('',)*(df.columns.nlevels - 1)
+        try:
+            integer_indexing = _labconfig.getboolean('lyse', 'integer_indexing')
+        except (LabConfig.NoOptionError, LabConfig.NoSectionError):
+            integer_indexing = False
+        if integer_indexing:
+            out = df.set_index(['sequence_index', 'run number', 'run repeat'], inplace=inplace, drop=False)
+            # out is None if inplace is True, and is the new dataframe is inplace is False.
+            if not inplace:
+                df = out
+        else:
+            out = df.set_index([('sequence',) + padding,('run time',) + padding], inplace=inplace, drop=False)
+            if not inplace:
+                df = out
+            df.index.names = ['sequence', 'run time']
+    except KeyError:
+        # Empty DataFrame or index column not found, so fall back to RangeIndex instead
+        pass
+    return df
+
 def globals_diff(run1, run2, group=None):
     return dict_diff(run1.get_globals(group), run2.get_globals(group))
  
diff --git a/lyse/__main__.py b/lyse/__main__.py
index 4c6e90d..cd6ea1f 100644
--- a/lyse/__main__.py
+++ b/lyse/__main__.py
@@ -52,7 +52,7 @@
 from qtutils import inmain_decorator, inmain, UiLoader, DisconnectContextManager
 from qtutils.auto_scroll_to_end import set_auto_scroll_to_end
 import qtutils.icons
-from lyse import LYSE_DIR
+from lyse import LYSE_DIR, _rangeindex_to_multiindex
 
 process_tree = ProcessTree.instance()
 
@@ -184,6 +184,7 @@ def handler(self, request_data):
             # sending the dataframe to a client requesting it, as we're doing now.
             app.filebox.shots_model.infer_objects()
             df = app.filebox.shots_model.dataframe
+            df = _rangeindex_to_multiindex(df, inplace=False)
 
             # Return only a subset of the dataframe if instructed to do so.
             if n_sequences is not None:

From b344e9ed3ba45798d6ecfab0839183546742fb01 Mon Sep 17 00:00:00 2001
From: Zak V <zakven@mit.edu>
Date: Thu, 29 Oct 2020 23:57:09 -0400
Subject: [PATCH 09/15] Simplified lyse.data() communication and made it
 backwards compatible.

---
 lyse/__init__.py | 29 +++++++++++++++++++----------
 lyse/__main__.py | 32 ++++++++++++--------------------
 2 files changed, 31 insertions(+), 30 deletions(-)

diff --git a/lyse/__init__.py b/lyse/__init__.py
index 3abf4fe..b270384 100644
--- a/lyse/__init__.py
+++ b/lyse/__init__.py
@@ -19,7 +19,6 @@
 import inspect
 import sys
 import threading
-import shlex
 
 import labscript_utils.h5_lock, h5py
 from labscript_utils.labconfig import LabConfig
@@ -145,25 +144,35 @@ def data(filepath=None, host='localhost', port=_lyse_port, timeout=5, n_sequence
     if filepath is not None:
         return _get_singleshot(filepath)
     else:
-        command_list = ['get_dataframe']
         if n_sequences is not None:
-            if type(n_sequences) is int and n_sequences >= 0:
-                command_list.append('--n_sequences {n_sequences}'.format(n_sequences=n_sequences))
-            else:
+            if not (type(n_sequences) is int and n_sequences >= 0):
                 msg = """n_sequences must be None or an integer greater than 0 but 
                     was {n_sequences}.""".format(n_sequences=n_sequences)
                 raise ValueError(dedent(msg))
         if filter_kwargs is not None:
-            if type(filter_kwargs) is dict:
-                command_list.append('--filter_kwargs ' + shlex.quote(repr(filter_kwargs)))
-            else:
+            if type(filter_kwargs) is not dict:
                 msg = """filter must be None or a dictionary but was 
                     {filter_kwargs}.""".format(filter_kwargs=filter_kwargs)
                 raise ValueError(dedent(msg))
-        command = ' '.join(command_list)
+
+        # Allow sending 'get dataframe' (without the enclosing list) if
+        # n_sequences and filter_kwargs aren't provided. This is for backwards
+        # compatability in case the server is running an outdated version of
+        # lyse.
+        if n_sequences is None and filter_kwargs is None:
+            command = 'get dataframe'
+        else:
+            command = ('get dataframe', n_sequences, filter_kwargs)
         df = zmq_get(port, host, command, timeout)
+        if isinstance(df, str) and df.startswith('error: operation not supported'):
+            # Sending a tuple for command to an outdated lyse servers causes it
+            # to reply with an error message.
+            msg = """The lyse server does not support n_sequences or filter_kwargs.
+                Call this function without providing those arguments to communicate
+                with this server."""
+            raise ValueError(dedent(msg))
         # Ensure conversion to multiindex is done, which needs to be done here
-        # if the server is running an old version of lyse.
+        # if the server is running an outdated version of lyse.
         _rangeindex_to_multiindex(df, inplace=True)
         df.sort_index(inplace=True)
         return df
diff --git a/lyse/__main__.py b/lyse/__main__.py
index cd6ea1f..25392ab 100644
--- a/lyse/__main__.py
+++ b/lyse/__main__.py
@@ -20,9 +20,6 @@
 import time
 import traceback
 import queue
-import argparse
-import shlex
-import ast
 
 # 3rd party imports:
 splash.update_text('importing numpy')
@@ -155,28 +152,13 @@ def get_screen_geometry():
 
 class WebServer(ZMQServer):
 
-    def __init__(self, *args, **kwargs):
-        ZMQServer.__init__(self, *args, **kwargs)
-        # Add parser for interpretting options for get_dataframe
-        parser = argparse.ArgumentParser()
-        parser.add_argument('--n_sequences', type=int)
-        parser.add_argument('--filter_kwargs', type=str)
-        self._command_parser = parser
-
     @inmain_decorator(wait_for_return=True)
     def handler(self, request_data):
         logger.info('WebServer request: %s' % str(request_data))
         if request_data == 'hello':
             return 'hello'
-        elif type(request_data) is str and request_data.startswith('get_dataframe'):
-            # Parse any arguments.
-            command_args = shlex.split(request_data)[1:]  # Strip leading 'get_dataframe'.
-            parsed_args = self._command_parser.parse_args(command_args)
-            n_sequences = parsed_args.n_sequences
-            filter_kwargs = parsed_args.filter_kwargs
-            if filter_kwargs is not None:
-                filter_kwargs = ast.literal_eval(filter_kwargs)
-
+        elif isinstance(request_data, tuple) and request_data[0]=='get dataframe' and len(request_data)==3:
+            _, n_sequences, filter_kwargs = request_data
             # infer_objects() picks fixed datatypes for columns that are compatible with
             # fixed datatypes, dramatically speeding up pickling. It is called here
             # rather than when updating the dataframe as calling it during updating may
@@ -200,6 +182,13 @@ def handler(self, request_data):
             # that no additional message is sent.
             self.send(df)
             return self.NO_RESPONSE
+        elif request_data == 'get dataframe':
+            # Ensure backwards compatability with clients using outdated
+            # versions of lyse.
+            app.filebox.shots_model.infer_objects()
+            df = app.filebox.shots_model.dataframe
+            self.send(df)
+            return self.NO_RESPONSE
         elif isinstance(request_data, dict):
             if 'filepath' in request_data:
                 h5_filepath = shared_drive.path_to_local(request_data['filepath'])
@@ -214,6 +203,9 @@ def handler(self, request_data):
             app.filebox.incoming_queue.put(shared_drive.path_to_local(request_data))
             return "Experiment added successfully\n"
 
+        return ("error: operation not supported. Recognised requests are:\n "
+                "'get dataframe'\n 'hello'\n {'filepath': <some_h5_filepath>}")
+
     def _extract_n_sequences_from_df(self, df, n_sequences):
         # If the dataframe is empty, just return it, otherwise accessing columns
         # below will raise a KeyError.

From f26aba2871d4f0de6e6ee830cefd3005136fc806 Mon Sep 17 00:00:00 2001
From: Zak V <zakven@mit.edu>
Date: Fri, 30 Oct 2020 12:32:55 -0400
Subject: [PATCH 10/15] Moved df.sort_index() from data() to
 _rangeindex_to_multiindex().

---
 lyse/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lyse/__init__.py b/lyse/__init__.py
index b270384..34e8fc8 100644
--- a/lyse/__init__.py
+++ b/lyse/__init__.py
@@ -174,7 +174,6 @@ def data(filepath=None, host='localhost', port=_lyse_port, timeout=5, n_sequence
         # Ensure conversion to multiindex is done, which needs to be done here
         # if the server is running an outdated version of lyse.
         _rangeindex_to_multiindex(df, inplace=True)
-        df.sort_index(inplace=True)
         return df
 
 def _rangeindex_to_multiindex(df, inplace):
@@ -200,6 +199,7 @@ def _rangeindex_to_multiindex(df, inplace):
     except KeyError:
         # Empty DataFrame or index column not found, so fall back to RangeIndex instead
         pass
+    df.sort_index(inplace=True)
     return df
 
 def globals_diff(run1, run2, group=None):

From 4c944dd85715d80ab66f651c927db932d61b4d43 Mon Sep 17 00:00:00 2001
From: Zak V <zakven@mit.edu>
Date: Fri, 30 Oct 2020 12:43:37 -0400
Subject: [PATCH 11/15] Updated the error message raised in data() when the
 server is running an outdated version of lyse.

---
 lyse/__init__.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lyse/__init__.py b/lyse/__init__.py
index 34e8fc8..5782ae9 100644
--- a/lyse/__init__.py
+++ b/lyse/__init__.py
@@ -169,7 +169,8 @@ def data(filepath=None, host='localhost', port=_lyse_port, timeout=5, n_sequence
             # to reply with an error message.
             msg = """The lyse server does not support n_sequences or filter_kwargs.
                 Call this function without providing those arguments to communicate
-                with this server."""
+                with this server, or upgrade the version of lyse running on the
+                server."""
             raise ValueError(dedent(msg))
         # Ensure conversion to multiindex is done, which needs to be done here
         # if the server is running an outdated version of lyse.

From 8d2a69ecc7455f4635172426d9df1441a4e349f1 Mon Sep 17 00:00:00 2001
From: Zak V <zakven@mit.edu>
Date: Fri, 30 Oct 2020 20:35:03 -0400
Subject: [PATCH 12/15] Added intersphinx link for Dataframe.filter() in
 data()'s docstring.

---
 lyse/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lyse/__init__.py b/lyse/__init__.py
index 5782ae9..8873fda 100644
--- a/lyse/__init__.py
+++ b/lyse/__init__.py
@@ -125,8 +125,8 @@ def data(filepath=None, host='localhost', port=_lyse_port, timeout=5, n_sequence
             returned. For example to call `filter()` with `like='temperature'`,
             set `filter_kwargs` to `{'like':'temperature'}`. If set to `None`
             then `Dataframe.filter()` will not be called. See
-            `Dataframe.filter()`'s documentation for more information. Defaults
-            to `None`.
+            :meth:`pandas:pandas.DataFrame.filter` for more information.
+            Defaults to `None`.
 
     Raises:
         ValueError: If `n_sequences` isn't `None` or a nonnegative integer, then

From d79bada47b64279defdb928f039e556c10bd31a8 Mon Sep 17 00:00:00 2001
From: Zak V <zakven@mit.edu>
Date: Sat, 31 Oct 2020 16:47:01 -0400
Subject: [PATCH 13/15] Reduced the amount of code run with @inmain_decorator()
 in WebServer.handler().

---
 lyse/__main__.py | 38 +++++++++++++++-----------------------
 1 file changed, 15 insertions(+), 23 deletions(-)

diff --git a/lyse/__main__.py b/lyse/__main__.py
index 25392ab..2d67df0 100644
--- a/lyse/__main__.py
+++ b/lyse/__main__.py
@@ -152,43 +152,24 @@ def get_screen_geometry():
 
 class WebServer(ZMQServer):
 
-    @inmain_decorator(wait_for_return=True)
     def handler(self, request_data):
         logger.info('WebServer request: %s' % str(request_data))
         if request_data == 'hello':
             return 'hello'
         elif isinstance(request_data, tuple) and request_data[0]=='get dataframe' and len(request_data)==3:
             _, n_sequences, filter_kwargs = request_data
-            # infer_objects() picks fixed datatypes for columns that are compatible with
-            # fixed datatypes, dramatically speeding up pickling. It is called here
-            # rather than when updating the dataframe as calling it during updating may
-            # call it needlessly often, whereas it only needs to be called prior to
-            # sending the dataframe to a client requesting it, as we're doing now.
-            app.filebox.shots_model.infer_objects()
-            df = app.filebox.shots_model.dataframe
-            df = _rangeindex_to_multiindex(df, inplace=False)
-
+            df = self._retrieve_dataframe()
+            df = _rangeindex_to_multiindex(df, inplace=True)
             # Return only a subset of the dataframe if instructed to do so.
             if n_sequences is not None:
                 df = self._extract_n_sequences_from_df(df, n_sequences)
             if filter_kwargs is not None:
                 df = df.filter(**filter_kwargs)
-            
-            # Returning the dataframe would mean that another thread would
-            # pickle/send it, which could happen partway through changes to the
-            # dataframe in the main thread. Sending it here prevents that thanks
-            # to the inmain_decorator (assuming that all modifications to the
-            # dataframe happen in the main thread). Then return NO_RESPONSE so
-            # that no additional message is sent.
-            self.send(df)
-            return self.NO_RESPONSE
+            return df
         elif request_data == 'get dataframe':
             # Ensure backwards compatability with clients using outdated
             # versions of lyse.
-            app.filebox.shots_model.infer_objects()
-            df = app.filebox.shots_model.dataframe
-            self.send(df)
-            return self.NO_RESPONSE
+            return self._retrieve_dataframe()
         elif isinstance(request_data, dict):
             if 'filepath' in request_data:
                 h5_filepath = shared_drive.path_to_local(request_data['filepath'])
@@ -206,6 +187,17 @@ def handler(self, request_data):
         return ("error: operation not supported. Recognised requests are:\n "
                 "'get dataframe'\n 'hello'\n {'filepath': <some_h5_filepath>}")
 
+    @inmain_decorator(wait_for_return=True)
+    def _retrieve_dataframe(self):
+        # infer_objects() picks fixed datatypes for columns that are compatible with
+        # fixed datatypes, dramatically speeding up pickling. It is called here
+        # rather than when updating the dataframe as calling it during updating may
+        # call it needlessly often, whereas it only needs to be called prior to
+        # sending the dataframe to a client requesting it, as we're doing now.
+        app.filebox.shots_model.infer_objects()
+        df = app.filebox.shots_model.dataframe.copy(deep=True)
+        return df
+
     def _extract_n_sequences_from_df(self, df, n_sequences):
         # If the dataframe is empty, just return it, otherwise accessing columns
         # below will raise a KeyError.

From bff7d19f58b70b83417503241619f527e4ddbb9d Mon Sep 17 00:00:00 2001
From: Zak V <zakven@mit.edu>
Date: Mon, 2 Nov 2020 10:41:33 -0500
Subject: [PATCH 14/15] Fixed some formatting in data()'s docstring.

---
 lyse/__init__.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lyse/__init__.py b/lyse/__init__.py
index 8873fda..d922c65 100644
--- a/lyse/__init__.py
+++ b/lyse/__init__.py
@@ -136,10 +136,10 @@ def data(filepath=None, host='localhost', port=_lyse_port, timeout=5, n_sequence
             lyse dataframe is returned.
 
     Returns:
-        (pandas series or dataframe): If `filepath` is provided, then a pandas
-            series with the data read from that file is returned. If `filepath`
-            is omitted or set to `None` then the lyse dataframe, or a subset of
-            it, is returned.
+        :obj:`pandas:pandas.DataFrame` or :obj:`pandas:pandas.Series`: If
+            `filepath` is provided, then a pandas series with the data read from
+            that file is returned. If `filepath` is omitted or set to `None`
+            then the lyse dataframe, or a subset of it, is returned.
     """    
     if filepath is not None:
         return _get_singleshot(filepath)

From f80f182959f874c570a95e62a75c3a8bbb28c53a Mon Sep 17 00:00:00 2001
From: Zak V <zakven@mit.edu>
Date: Tue, 3 Nov 2020 20:21:38 -0500
Subject: [PATCH 15/15] Corrected formatting in data()'s docstring.

---
 lyse/__init__.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lyse/__init__.py b/lyse/__init__.py
index d922c65..cf895e0 100644
--- a/lyse/__init__.py
+++ b/lyse/__init__.py
@@ -137,9 +137,9 @@ def data(filepath=None, host='localhost', port=_lyse_port, timeout=5, n_sequence
 
     Returns:
         :obj:`pandas:pandas.DataFrame` or :obj:`pandas:pandas.Series`: If
-            `filepath` is provided, then a pandas series with the data read from
-            that file is returned. If `filepath` is omitted or set to `None`
-            then the lyse dataframe, or a subset of it, is returned.
+        `filepath` is provided, then a pandas series with the data read from
+        that file is returned. If `filepath` is omitted or set to `None` then
+        the lyse dataframe, or a subset of it, is returned.
     """    
     if filepath is not None:
         return _get_singleshot(filepath)