From 488c22b9492cda4870999acedd2c28d9341e5c3b Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Fri, 27 Jan 2017 08:25:58 -0800 Subject: [PATCH 1/2] Add global disable_list_exceptions variable to disable exceptions thrown during expensive operations. Raise ListError if mapreduce over a bucket is attempted --- Makefile | 2 ++ riak/__init__.py | 12 +++++++++--- riak/client/operations.py | 23 +++++++++++++++++++++-- riak/mapreduce.py | 17 ++++++++++------- riak/riak_error.py | 11 +++++++++++ riak/tests/base.py | 3 +++ riak/tests/test_kv.py | 34 ++++++++++++++++++++++++++++++++-- riak/tests/test_mapreduce.py | 9 ++++++++- riak/tests/yz_setup.py | 4 ++++ 9 files changed, 100 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 316389e4..166e4007 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,8 @@ DOCSDIR := $(PROJDIR)/docs PYPI_REPOSITORY ?= pypi +all: lint test + .PHONY: lint lint: $(PROJDIR)/.runner lint diff --git a/riak/__init__.py b/riak/__init__.py index de68354a..306cf7a0 100644 --- a/riak/__init__.py +++ b/riak/__init__.py @@ -19,7 +19,7 @@ operations, and run Linkwalking operations. """ -from riak.riak_error import RiakError, ConflictError +from riak.riak_error import RiakError, ConflictError, ListError from riak.client import RiakClient from riak.bucket import RiakBucket, BucketType from riak.table import Table @@ -30,11 +30,17 @@ __all__ = ['RiakBucket', 'Table', 'BucketType', 'RiakNode', 'RiakObject', 'RiakClient', 'RiakMapReduce', 'RiakKeyFilter', - 'RiakLink', 'RiakError', 'ConflictError', - 'ONE', 'ALL', 'QUORUM', 'key_filter'] + 'RiakLink', 'RiakError', 'ConflictError', 'ListError', + 'ONE', 'ALL', 'QUORUM', 'key_filter', + 'disable_list_exceptions'] ONE = "one" ALL = "all" QUORUM = "quorum" key_filter = RiakKeyFilter() + +""" +Set to true to allow listing operations +""" +disable_list_exceptions = False diff --git a/riak/client/operations.py b/riak/client/operations.py index 1acf06e1..0d507f12 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -13,11 +13,11 @@ # limitations under the License. import six - import riak.client.multi +from riak import ListError from riak.client.transport import RiakClientTransport, \ - retryable, retryableHttpOnly + retryable, retryableHttpOnly from riak.client.index_page import IndexPage from riak.datatypes import TYPES from riak.table import Table @@ -55,7 +55,11 @@ def get_buckets(self, transport, bucket_type=None, timeout=None): :rtype: list of :class:`RiakBucket ` instances """ + if not riak.disable_list_exceptions: + raise ListError() + _validate_timeout(timeout) + if bucket_type: bucketfn = self._bucket_type_bucket_builder else: @@ -100,6 +104,9 @@ def stream_buckets(self, bucket_type=None, timeout=None): ` instances """ + if not riak.disable_list_exceptions: + raise ListError() + _validate_timeout(timeout) if bucket_type: @@ -467,7 +474,11 @@ def get_keys(self, transport, bucket, timeout=None): :type timeout: int :rtype: list """ + if not riak.disable_list_exceptions: + raise ListError() + _validate_timeout(timeout) + return transport.get_keys(bucket, timeout=timeout) def stream_keys(self, bucket, timeout=None): @@ -503,6 +514,9 @@ def stream_keys(self, bucket, timeout=None): :type timeout: int :rtype: iterator """ + if not riak.disable_list_exceptions: + raise ListError() + _validate_timeout(timeout) def make_op(transport): @@ -678,10 +692,15 @@ def ts_stream_keys(self, table, timeout=None): :type timeout: int :rtype: iterator """ + if not riak.disable_list_exceptions: + raise ListError() + t = table if isinstance(t, six.string_types): t = Table(self, table) + _validate_timeout(timeout) + resource = self._acquire() transport = resource.object stream = transport.ts_stream_keys(t, timeout) diff --git a/riak/mapreduce.py b/riak/mapreduce.py index 7d2f690e..1b604663 100644 --- a/riak/mapreduce.py +++ b/riak/mapreduce.py @@ -17,9 +17,9 @@ from __future__ import print_function from collections import Iterable, namedtuple -from riak import RiakError from six import string_types, PY2 -from riak.bucket import RiakBucket + +import riak #: Links are just bucket/key/tag tuples, this class provides a @@ -128,8 +128,10 @@ def add_bucket(self, bucket, bucket_type=None): :type bucket_type: string, None :rtype: :class:`RiakMapReduce` """ + if not riak.disable_list_exceptions: + raise riak.ListError() self._input_mode = 'bucket' - if isinstance(bucket, RiakBucket): + if isinstance(bucket, riak.RiakBucket): if bucket.bucket_type.is_default(): self._inputs = {'bucket': bucket.name} else: @@ -308,14 +310,15 @@ def run(self, timeout=None): try: result = self._client.mapred(self._inputs, query, timeout) - except RiakError as e: + except riak.RiakError as e: if 'worker_startup_failed' in e.value: for phase in self._phases: if phase._language == 'erlang': if type(phase._function) is str: - raise RiakError('May have tried erlang strfun ' - 'when not allowed\n' - 'original error: ' + e.value) + raise riak.RiakError( + 'May have tried erlang strfun ' + 'when not allowed\n' + 'original error: ' + e.value) raise e # If the last phase is NOT a link phase, then return the result. diff --git a/riak/riak_error.py b/riak/riak_error.py index 97d0878c..4fe0ce05 100644 --- a/riak/riak_error.py +++ b/riak/riak_error.py @@ -36,3 +36,14 @@ class ConflictError(RiakError): """ def __init__(self, message='Object in conflict'): super(ConflictError, self).__init__(message) + + +class ListError(RiakError): + """ + Raised when a list operation is attempted and + riak.disable_list_exceptions is false. + """ + def __init__(self, message='Bucket and key list operations ' + 'are expensive and should not be ' + 'used in production.'): + super(ListError, self).__init__(message) diff --git a/riak/tests/base.py b/riak/tests/base.py index aa81c0da..9aaf4e69 100644 --- a/riak/tests/base.py +++ b/riak/tests/base.py @@ -15,6 +15,7 @@ # -*- coding: utf-8 -*- import logging import random +import riak from riak.client import RiakClient from riak.tests import HOST, PROTOCOL, PB_PORT, HTTP_PORT, SECURITY_CREDS @@ -70,9 +71,11 @@ def create_client(cls, host=None, http_port=None, pb_port=None, **kwargs) def setUp(self): + riak.disable_list_exceptions = True self.bucket_name = self.randname() self.key_name = self.randname() self.client = self.create_client() def tearDown(self): + riak.disable_list_exceptions = False self.client.close() diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 56f55844..63206c95 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -20,7 +20,8 @@ from six import string_types, PY2, PY3 from time import sleep -from riak import ConflictError, RiakBucket, RiakError +from riak import ConflictError, RiakError, ListError +from riak import RiakClient, RiakBucket, BucketType from riak.resolver import default_resolver, last_written_resolver from riak.tests import RUN_KV, RUN_RESOLVE, PROTOCOL from riak.tests.base import IntegrationTestBase @@ -63,7 +64,6 @@ def tearDownModule(): class NotJsonSerializable(object): - def __init__(self, *args, **kwargs): self.args = list(args) self.kwargs = kwargs @@ -86,6 +86,36 @@ def __eq__(self, other): return True +class KVUnitTests(unittest.TestCase): + def test_list_keys_exception(self): + c = RiakClient() + bt = BucketType(c, 'test') + b = RiakBucket(c, 'test', bt) + with self.assertRaises(ListError): + b.get_keys() + + def test_stream_buckets_exception(self): + c = RiakClient() + with self.assertRaises(ListError): + bs = [] + for bl in c.stream_buckets(): + bs.extend(bl) + + def test_stream_keys_exception(self): + c = RiakClient() + with self.assertRaises(ListError): + ks = [] + for kl in c.stream_keys('test'): + ks.extend(kl) + + def test_ts_stream_keys_exception(self): + c = RiakClient() + with self.assertRaises(ListError): + ks = [] + for kl in c.ts_stream_keys('test'): + ks.extend(kl) + + @unittest.skipUnless(RUN_KV, 'RUN_KV is 0') class BasicKVTests(IntegrationTestBase, unittest.TestCase, Comparison): def test_no_returnbody(self): diff --git a/riak/tests/test_mapreduce.py b/riak/tests/test_mapreduce.py index f8b90e98..bfdfc7dd 100644 --- a/riak/tests/test_mapreduce.py +++ b/riak/tests/test_mapreduce.py @@ -19,7 +19,7 @@ from six import PY2 from riak.mapreduce import RiakMapReduce -from riak import key_filter, RiakError +from riak import key_filter, RiakClient, RiakError, ListError from riak.tests import RUN_MAPREDUCE, RUN_SECURITY, RUN_YZ from riak.tests.base import IntegrationTestBase from riak.tests.test_yokozuna import wait_for_yz_index @@ -39,6 +39,13 @@ def tearDownModule(): yzTearDown(testrun_yz_mr) +class MapReduceUnitTests(unittest.TestCase): + def test_mapred_bucket_exception(self): + c = RiakClient() + with self.assertRaises(ListError): + c.add('bucket') + + @unittest.skipUnless(RUN_MAPREDUCE, 'RUN_MAPREDUCE is 0') class LinkTests(IntegrationTestBase, unittest.TestCase): def test_store_and_get_links(self): diff --git a/riak/tests/yz_setup.py b/riak/tests/yz_setup.py index 78c44755..88a7daee 100644 --- a/riak/tests/yz_setup.py +++ b/riak/tests/yz_setup.py @@ -13,6 +13,7 @@ # limitations under the License. import logging +import riak from riak import RiakError from riak.tests import RUN_YZ @@ -21,6 +22,7 @@ def yzSetUp(*yzdata): if RUN_YZ: + riak.disable_list_exceptions = True c = IntegrationTestBase.create_client() for yz in yzdata: logging.debug("yzSetUp: %s", yz) @@ -43,6 +45,7 @@ def yzSetUp(*yzdata): def yzTearDown(c, *yzdata): if RUN_YZ: + riak.disable_list_exceptions = True c = IntegrationTestBase.create_client() for yz in yzdata: logging.debug("yzTearDown: %s", yz) @@ -57,3 +60,4 @@ def yzTearDown(c, *yzdata): for key in keys: b.delete(key) c.close() + riak.disable_list_exceptions = False From 4fee03a6c0fd51a368c0364adcc5408dafe9c003 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Thu, 9 Feb 2017 08:43:56 -0800 Subject: [PATCH 2/2] Add note about exceptions being raised for list operations --- RELNOTES.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/RELNOTES.md b/RELNOTES.md index bd567fde..0cc58993 100644 --- a/RELNOTES.md +++ b/RELNOTES.md @@ -1,5 +1,9 @@ # Riak Python Client Release Notes +## [`2.8.0` Release](https://github.com/basho/riak-python-client/issues?q=milestone%3Ariak-python-client-2.8.0) + +* [Running expensive operations *now raise exceptions*](https://github.com/basho/riak-python-client/pull/518). You can disable these exceptions for development purposes but should not do so in production. + ## [`2.7.0` Release](https://github.com/basho/riak-python-client/issues?q=milestone%3Ariak-python-client-2.7.0) * Riak TS 1.5 support * Support for `head` parameter