From ebef628efd74df0fd1c91a1514ef60c3e448fcb5 Mon Sep 17 00:00:00 2001 From: Teodoro Montanaro Date: Tue, 12 May 2020 12:53:58 +0200 Subject: [PATCH 1/3] execute_with_columns method was added --- README.md | 17 ++++--------- sqlite3worker.py | 62 ++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 64 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index e5c4abf..fbdd013 100644 --- a/README.md +++ b/README.md @@ -17,19 +17,6 @@ Installation is via the usual ``setup.py`` method: sudo python setup.py install ``` -Alternatively one can use ``pip`` to install directly from the git repository -without having to clone first: - -```sh -sudo pip install git+https://github.com/palantir/sqlite3worker#egg=sqlite3worker -``` - -One may also use ``pip`` to install on a per-user basis without requiring -super-user permissions: - -```sh -pip install --user git+https://github.com/palantir/sqlite3worker#egg=sqlite3worker -``` ## Example ```python @@ -44,6 +31,10 @@ results = sql_worker.execute("SELECT * from tester") for timestamp, uuid in results: print(timestamp, uuid) +results, columns = sql_worker.execute_with_columns("SELECT * from tester") +for timestamp, uuid in results: + print(timestamp, uuid) + sql_worker.close() ``` diff --git a/sqlite3worker.py b/sqlite3worker.py index 23eec87..e57a2f5 100644 --- a/sqlite3worker.py +++ b/sqlite3worker.py @@ -18,10 +18,13 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. +# Teodoro Montanaro only added the "execute_with_columns" and "query_results_with_columns" methods +# So now, by using "execute_with_columns" method instead of "execute" it is possible to obtain as a result of the select query the following structure: (rows, columns) + """Thread safe sqlite3 interface.""" -__author__ = "Shawn Lee" -__email__ = "shawnl@palantir.com" +__author__ = "Shawn Lee, Teodoro Montanaro" +__email__ = "shawnl@palantir.com, teodoro.montanaro@linksfoundation.com" __license__ = "MIT" import logging @@ -67,6 +70,7 @@ def __init__(self, file_name, max_queue_size=100): self.sqlite3_cursor = self.sqlite3_conn.cursor() self.sql_queue = Queue.Queue(maxsize=max_queue_size) self.results = {} + self.columns = {} self.max_queue_size = max_queue_size self.exit_set = False # Token that is put into queue when close() is called. @@ -128,6 +132,10 @@ def run_query(self, token, query, values): "Query returned error: %s: %s: %s" % (query, values, err)) LOGGER.error( "Query returned error: %s: %s: %s", query, values, err) + try: + self.columns[token] = list(map(lambda x: x[0], self.sqlite3_cursor.description)) + except: + LOGGER.error("It was not possible to retrieve columns' names") else: try: self.sqlite3_cursor.execute(query, values) @@ -171,6 +179,31 @@ def query_results(self, token): if delay < 8: delay += delay + def query_results_with_columns(self, token): + """Get the query results for a specific token. + + Args: + token: A uuid object of the query you want returned. + + Returns: + Return the results of the query when it's executed by the thread. + """ + delay = .001 + while True: + if token in self.results: + return_val = self.results[token] + return_col = self.columns[token] + del self.results[token] + del self.columns[token] + return (return_val, return_col) + # Double back on the delay to a max of 8 seconds. This prevents + # a long lived select statement from trashing the CPU with this + # infinite loop as it's waiting for the query results. + LOGGER.debug("Sleeping: %s %s", delay, token) + time.sleep(delay) + if delay < 8: + delay += delay + def execute(self, query, values=None): """Execute a query. @@ -195,3 +228,28 @@ def execute(self, query, values=None): return self.query_results(token) else: self.sql_queue.put((token, query, values), timeout=5) + + def execute_with_columns(self, query, values=None): + """Execute a query. + + Args: + query: The sql string using ? for placeholders of dynamic values. + values: A tuple of values to be replaced into the ? of the query. + + Returns: + If it's a select query it will return the results of the query. + """ + if self.exit_set: + LOGGER.debug("Exit set, not running: %s", query) + return "Exit Called" + LOGGER.debug("execute: %s", query) + values = values or [] + # A token to track this query with. + token = str(uuid.uuid4()) + # If it's a select we queue it up with a token to mark the results + # into the output queue so we know what results are ours. + if query.lower().strip().startswith("select"): + self.sql_queue.put((token, query, values), timeout=5) + return self.query_results_with_columns(token) + else: + self.sql_queue.put((token, query, values), timeout=5) \ No newline at end of file From fabc1e092f0df9432b13eb95f8101f87ddb1d63b Mon Sep 17 00:00:00 2001 From: Teodoro Montanaro Date: Tue, 12 May 2020 12:57:34 +0200 Subject: [PATCH 2/3] minor changes to readme --- README.md | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index fbdd013..6941447 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,13 @@ -# Sqlite3Worker +# Fork of Sqlite3Worker -A threadsafe sqlite worker. +Sqlite3Worker is a threadsafe sqlite worker. +In this fork we added the possibility to retrieve the columns' names (heading) from the query by using the "execute_with_columns" method instead of the "execute" one. +To do it you can use the following structure: + ``` + rows, columns = sql_worker.execute_with_columns("SELECT * from tester") + ``` -This library implements a thread pool pattern with sqlite3 being the desired +This library (thanks to the one from which it was forked) implements a thread pool pattern with sqlite3 being the desired output. sqllite3 implementation lacks the ability to safely modify the sqlite3 database From 18a01483de24c3664cb0400f7c7ec5f0a525784a Mon Sep 17 00:00:00 2001 From: Teodoro Montanaro Date: Tue, 12 May 2020 12:58:03 +0200 Subject: [PATCH 3/3] minor changes to readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6941447..e42d169 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Fork of Sqlite3Worker -Sqlite3Worker is a threadsafe sqlite worker. +Sqlite3Worker is a threadsafe sqlite worker. In this fork we added the possibility to retrieve the columns' names (heading) from the query by using the "execute_with_columns" method instead of the "execute" one. To do it you can use the following structure: ```