From b3c4c03315d27e128f8814f67ee7d2447fbc76e5 Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Mon, 5 Oct 2015 15:14:50 +0000 Subject: [PATCH 1/8] Add support for fetching the bucket/key preflist --- riak/bucket.py | 10 ++++++++++ riak/client/operations.py | 17 +++++++++++++++++ riak/tests/test_kv.py | 11 +++++++++++ riak/transports/feature_detect.py | 12 +++++++++++- riak/transports/http/resources.py | 28 ++++++++++++++++++++++++++++ riak/transports/http/transport.py | 20 ++++++++++++++++++++ riak/transports/pbc/codec.py | 13 +++++++++++++ riak/transports/pbc/transport.py | 23 +++++++++++++++++++++++ riak/transports/transport.py | 6 ++++++ 9 files changed, 139 insertions(+), 1 deletion(-) diff --git a/riak/bucket.py b/riak/bucket.py index bb95726d..4342d7ad 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -586,6 +586,16 @@ def update_counter(self, key, value, **kwargs): increment_counter = update_counter + def get_preflist(self, key): + """ + Retrieve the preflist associated with a given bucket/key + + :param key: Name of the key. + :type key: string + :rtype: list of dict() + """ + return self._client.get_preflist(self, key) + def __str__(self): if self.bucket_type.is_default(): return ''.format(self.name) diff --git a/riak/client/operations.py b/riak/client/operations.py index ecda64af..8239e4ac 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -1000,6 +1000,23 @@ def update_datatype(self, datatype, w=None, dw=None, pw=None, timeout=timeout, include_context=include_context) + @retryable + def get_preflist(self, transport, bucket, key): + """ + Fetch the preflist for a given bucket and key. + + .. note:: This request is automatically retried :attr:`retries` + times if it fails due to network error. + + :param bucket: the bucket whose index will be queried + :type bucket: RiakBucket + :param key: the key of the preflist + :type key: string + + :return: list of dicts (partition, node, primary) + """ + return transport.get_preflist(bucket, key) + def _bucket_type_bucket_builder(self, name, bucket_type): """ Build a bucket from a bucket type diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index b2aa9f92..4306cc30 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -579,6 +579,17 @@ def test_get_params(self): basic_quorum=True) self.assertFalse(missing.exists) + def test_preflist(self): + bucket = self.client.bucket(self.bucket_name) + bucket.new(self.key_name, data={"foo": "one", + "bar": "baz"}).store() + preflist = bucket.get_preflist(self.key_name) + preflist2 = self.client.get_preflist(bucket, self.key_name) + for pref in (preflist, preflist2): + self.assertEqual(len(pref), 3) + self.assertEqual(pref[0]['node'], 'riak@127.0.0.1') + [self.assertTrue(node['primary']) for node in pref] + def generate_siblings(self, original, count=5, delay=None): vals = [] for _ in range(count): diff --git a/riak/transports/feature_detect.py b/riak/transports/feature_detect.py index e860a4c6..642fac96 100644 --- a/riak/transports/feature_detect.py +++ b/riak/transports/feature_detect.py @@ -26,7 +26,9 @@ 1.2: LooseVersion("1.2.0"), 1.4: LooseVersion("1.4.0"), 1.44: LooseVersion("1.4.4"), - 2.0: LooseVersion("2.0.0") + 2.0: LooseVersion("2.0.0"), + 2.1: LooseVersion("2.1.0"), + 2.12: LooseVersion("2.1.2") } @@ -192,6 +194,14 @@ def datatypes(self): """ return self.server_version >= versions[2.0] + def preflists(self): + """ + Whether bucket/key preflists are supported. + + :rtype: bool + """ + return self.server_version >= versions[2.1] + @lazy_property def server_version(self): return LooseVersion(self._server_version()) diff --git a/riak/transports/http/resources.py b/riak/transports/http/resources.py index f30cb36e..c13925bc 100644 --- a/riak/transports/http/resources.py +++ b/riak/transports/http/resources.py @@ -172,6 +172,30 @@ def datatypes_path(self, bucket_type, bucket, key=None, **options): return mkpath("/types", quote_plus(bucket_type), "buckets", quote_plus(bucket), "datatypes", key, **options) + def preflist_path(self, bucket, key, bucket_type=None, **options): + """ + Generate the URL for bucket/key preflist information + + :param bucket: Name of a Riak bucket + :type bucket: string + :param key: Name of a Key + :type key: string + :param bucket_type: Optional Riak Bucket Type + :type bucket_type: None or string + :rtype URL string + """ + if not self.riak_kv_wm_preflist: + raise RiakError("Preflists are unsupported by this Riak node") + if self.riak_kv_wm_bucket_type and bucket_type: + return mkpath("/types", quote_plus(bucket_type), + "buckets", quote_plus(bucket), + "keys", quote_plus(key), + "preflist", **options) + else: + return mkpath("/buckets", quote_plus(bucket), + "keys", quote_plus(key), + "preflist", **options) + # Feature detection overrides def bucket_types(self): return self.riak_kv_wm_bucket_type is not None @@ -225,6 +249,10 @@ def riak_solr_indexer_wm(self): def riak_kv_wm_counter(self): return self.resources.get('riak_kv_wm_counter') + @lazy_property + def riak_kv_wm_preflist(self): + return self.resources.get('riak_kv_wm_preflist') + @lazy_property def yz_wm_search(self): return self.resources.get('yz_wm_search') diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index f534310a..aaac3f92 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -775,6 +775,26 @@ def update_datatype(self, datatype, **options): return True + def get_preflist(self, bucket, key): + """ + Get the preflist for a bucket/key + + :param bucket: Riak Bucket + :type bucket: :class:`~riak.bucket.RiakBucket` + :param key: Riak Key + :type key: string + :rtype: list of dicts + """ + bucket_type = self._get_bucket_type(bucket.bucket_type) + url = self.preflist_path(bucket.name, key, bucket_type=bucket_type) + status, headers, body = self._request('GET', url) + + if status == 200: + preflist = json.loads(bytes_to_str(body)) + return preflist['preflist'] + else: + raise RiakError('Error getting bucket/key preflist.') + def check_http_code(self, status, expected_statuses): if status not in expected_statuses: raise RiakError('Expected status %s, received %s' % diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index a3f26e8a..1a2bdd1b 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -622,3 +622,16 @@ def _encode_map_update(self, dtype, msg, op): msg.flag_op = riak_pb.MapUpdate.ENABLE else: msg.flag_op = riak_pb.MapUpdate.DISABLE + + def _decode_preflist(self, item): + """ + Decodes a preflist response + + :param preflist: a bucket/key preflist + :type preflist: list of riak_pb.RpbBucketKeyPreflistItem + :rtype dict + """ + result = {'partition': item.partition, + 'node': bytes_to_str(item.node), + 'primary': item. primary} + return result diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index c77dab2b..5e527c51 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -56,6 +56,8 @@ MSG_CODE_SET_BUCKET_RESP, MSG_CODE_GET_BUCKET_TYPE_REQ, MSG_CODE_SET_BUCKET_TYPE_REQ, + MSG_CODE_GET_BUCKET_KEY_PREFLIST_REQ, + MSG_CODE_GET_BUCKET_KEY_PREFLIST_RESP, MSG_CODE_MAP_RED_REQ, MSG_CODE_INDEX_REQ, MSG_CODE_INDEX_RESP, @@ -699,3 +701,24 @@ def update_datatype(self, datatype, **options): datatype._set_value(self._decode_dt_value(type_name, resp)) return True + + def get_preflist(self, bucket, key): + """ + Get the preflist for a bucket/key + + :param bucket: Riak Bucket + :type bucket: :class:`~riak.bucket.RiakBucket` + :param key: Riak Key + :type key: string + :rtype: list of dicts + """ + req = riak_pb.RpbGetBucketKeyPreflistReq() + req.bucket = str_to_bytes(bucket.name) + req.key = str_to_bytes(key) + req.type = str_to_bytes(bucket.bucket_type.name) + + msg_code, resp = self._request(MSG_CODE_GET_BUCKET_KEY_PREFLIST_REQ, + req, + MSG_CODE_GET_BUCKET_KEY_PREFLIST_RESP) + + return [self._decode_preflist(item) for item in resp.preflist] diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 85dcae43..be598511 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -270,6 +270,12 @@ def update_datatype(self, datatype, w=None, dw=None, pw=None, """ raise NotImplementedError + def get_preflist(self, bucket, key): + """ + Fetches the preflist for a bucket/key. + """ + raise NotImplementedError + def _search_mapred_emu(self, index, query): """ Emulates a search request via MapReduce. Used in the case From 315535220994bac197707e340beefe7a3349859e Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Sun, 11 Oct 2015 17:30:12 +0000 Subject: [PATCH 2/8] Add support for write-once bucket type property --- commands.py | 2 ++ riak/tests/test_btypes.py | 21 +++++++++++++++++++++ riak/transports/feature_detect.py | 8 ++++++++ riak/transports/pbc/codec.py | 3 ++- 4 files changed, 33 insertions(+), 1 deletion(-) diff --git a/commands.py b/commands.py index 1cf60eb5..c2afab0e 100644 --- a/commands.py +++ b/commands.py @@ -81,6 +81,7 @@ class create_bucket_types(Command): * `pytest-sets` with ``{"datatype":"set"}`` * `pytest-counters` with ``{"datatype":"counter"}`` * `pytest-consistent` with ``{"consistent":true}`` + * `pytest-write-once` with ``{"write_once": true}`` * `pytest-mr` * `pytest` with ``{"allow_mult":false}`` """ @@ -96,6 +97,7 @@ class create_bucket_types(Command): 'pytest-sets': {'datatype': 'set'}, 'pytest-counters': {'datatype': 'counter'}, 'pytest-consistent': {'consistent': True}, + 'pytest-write-once': {'write_once': True}, 'pytest-mr': {}, 'pytest': {'allow_mult': False} } diff --git a/riak/tests/test_btypes.py b/riak/tests/test_btypes.py index 89d298b3..b3ea5db2 100644 --- a/riak/tests/test_btypes.py +++ b/riak/tests/test_btypes.py @@ -176,3 +176,24 @@ def test_multiget_bucket_types(self): self.assertIsInstance(mobj, RiakObject) self.assertEqual(bucket, mobj.bucket) self.assertEqual(btype, mobj.bucket.bucket_type) + + @unittest.skipIf(SKIP_BTYPES == '1', "SKIP_BTYPES is set") + def test_write_once_bucket_type(self): + btype = self.client.bucket_type('pytest-write-once') + btype.set_property('write_once', True) + bucket = btype.bucket(self.bucket_name) + + for i in range(100): + obj = bucket.new(self.key_name + str(i)) + obj.data = {'id': i} + obj.store() + + mget = bucket.multiget([self.key_name + str(i) for i in range(100)]) + for mobj in mget: + self.assertIsInstance(mobj, RiakObject) + self.assertEqual(bucket, mobj.bucket) + self.assertEqual(btype, mobj.bucket.bucket_type) + + props = btype.get_properties() + self.assertIn('write_once', props) + self.assertEqual(True, props['write_once']) diff --git a/riak/transports/feature_detect.py b/riak/transports/feature_detect.py index 642fac96..c73ba37d 100644 --- a/riak/transports/feature_detect.py +++ b/riak/transports/feature_detect.py @@ -202,6 +202,14 @@ def preflists(self): """ return self.server_version >= versions[2.1] + def write_once(self): + """ + Whether write-once operations are supported. + + :rtype: bool + """ + return self.server_version >= versions[2.1] + @lazy_property def server_version(self): return LooseVersion(self._server_version()) diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index 1a2bdd1b..02c53ca5 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -51,7 +51,8 @@ def _invert(d): NORMAL_PROPS = ['n_val', 'allow_mult', 'last_write_wins', 'old_vclock', 'young_vclock', 'big_vclock', 'small_vclock', 'basic_quorum', - 'notfound_ok', 'search', 'backend', 'search_index', 'datatype'] + 'notfound_ok', 'search', 'backend', 'search_index', 'datatype', + 'write_once'] COMMIT_HOOK_PROPS = ['precommit', 'postcommit'] MODFUN_PROPS = ['chash_keyfun', 'linkfun'] QUORUM_PROPS = ['r', 'pr', 'w', 'pw', 'dw', 'rw'] From 63a205b67dd778eab248987bde4b093d96504b10 Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Sun, 11 Oct 2015 19:48:52 +0000 Subject: [PATCH 3/8] Update the feature detection tests for Riak 2.1 --- riak/tests/test_feature_detection.py | 35 ++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/riak/tests/test_feature_detection.py b/riak/tests/test_feature_detection.py index 11dadc75..682c5ac2 100644 --- a/riak/tests/test_feature_detection.py +++ b/riak/tests/test_feature_detection.py @@ -60,6 +60,8 @@ def test_pre_10(self): self.assertFalse(t.index_term_regex()) self.assertFalse(t.bucket_types()) self.assertFalse(t.datatypes()) + self.assertFalse(t.preflists()) + self.assertFalse(t.write_once()) def test_10(self): t = DummyTransport("1.0.3") @@ -77,6 +79,8 @@ def test_10(self): self.assertFalse(t.index_term_regex()) self.assertFalse(t.bucket_types()) self.assertFalse(t.datatypes()) + self.assertFalse(t.preflists()) + self.assertFalse(t.write_once()) def test_11(self): t = DummyTransport("1.1.4") @@ -94,6 +98,8 @@ def test_11(self): self.assertFalse(t.index_term_regex()) self.assertFalse(t.bucket_types()) self.assertFalse(t.datatypes()) + self.assertFalse(t.preflists()) + self.assertFalse(t.write_once()) def test_12(self): t = DummyTransport("1.2.0") @@ -111,6 +117,8 @@ def test_12(self): self.assertFalse(t.index_term_regex()) self.assertFalse(t.bucket_types()) self.assertFalse(t.datatypes()) + self.assertFalse(t.preflists()) + self.assertFalse(t.write_once()) def test_12_loose(self): t = DummyTransport("1.2.1p3") @@ -128,6 +136,8 @@ def test_12_loose(self): self.assertFalse(t.index_term_regex()) self.assertFalse(t.bucket_types()) self.assertFalse(t.datatypes()) + self.assertFalse(t.preflists()) + self.assertFalse(t.write_once()) def test_14(self): t = DummyTransport("1.4.0rc1") @@ -145,6 +155,8 @@ def test_14(self): self.assertFalse(t.index_term_regex()) self.assertFalse(t.bucket_types()) self.assertFalse(t.datatypes()) + self.assertFalse(t.preflists()) + self.assertFalse(t.write_once()) def test_144(self): t = DummyTransport("1.4.6") @@ -162,6 +174,8 @@ def test_144(self): self.assertTrue(t.index_term_regex()) self.assertFalse(t.bucket_types()) self.assertFalse(t.datatypes()) + self.assertFalse(t.preflists()) + self.assertFalse(t.write_once()) def test_20(self): t = DummyTransport("2.0.1") @@ -179,6 +193,27 @@ def test_20(self): self.assertTrue(t.index_term_regex()) self.assertTrue(t.bucket_types()) self.assertTrue(t.datatypes()) + self.assertFalse(t.preflists()) + self.assertFalse(t.write_once()) + + def test_21(self): + t = DummyTransport("2.1.0") + self.assertTrue(t.phaseless_mapred()) + self.assertTrue(t.pb_indexes()) + self.assertTrue(t.pb_search()) + self.assertTrue(t.pb_conditionals()) + self.assertTrue(t.quorum_controls()) + self.assertTrue(t.tombstone_vclocks()) + self.assertTrue(t.pb_head()) + self.assertTrue(t.pb_clear_bucket_props()) + self.assertTrue(t.pb_all_bucket_props()) + self.assertTrue(t.counters()) + self.assertTrue(t.stream_indexes()) + self.assertTrue(t.index_term_regex()) + self.assertTrue(t.bucket_types()) + self.assertTrue(t.datatypes()) + self.assertTrue(t.preflists()) + self.assertTrue(t.write_once()) if __name__ == '__main__': unittest.main() From 30f8ae09173aa7a50b83372f954b4878afc77f4c Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Tue, 13 Oct 2015 15:30:22 +0000 Subject: [PATCH 4/8] Add optional timeout to YZ create index command --- riak/client/operations.py | 7 +++++-- riak/tests/test_all.py | 2 +- riak/transports/http/transport.py | 7 ++++++- riak/transports/pbc/transport.py | 5 ++++- riak/transports/transport.py | 3 ++- 5 files changed, 18 insertions(+), 6 deletions(-) diff --git a/riak/client/operations.py b/riak/client/operations.py index 8239e4ac..b5972f69 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -685,7 +685,8 @@ def stream_mapred(self, inputs, query, timeout): stream.close() @retryable - def create_search_index(self, transport, index, schema=None, n_val=None): + def create_search_index(self, transport, index, schema=None, n_val=None, + timeout=None): """ create_search_index(index, schema=None, n_val=None) @@ -698,8 +699,10 @@ def create_search_index(self, transport, index, schema=None, n_val=None): :type schema: string, None :param n_val: this indexes N value :type n_val: integer, None + :param timeout: optional timeout (in ms) + :type timeout: integer, None """ - return transport.create_search_index(index, schema, n_val) + return transport.create_search_index(index, schema, n_val, timeout) @retryable def get_search_index(self, transport, index): diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 992e4997..2a6ef8cc 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -87,7 +87,7 @@ def setUpModule(): 'index': 'mrbucket'} for yz in (testrun_yz, testrun_yz_index, testrun_yz_mr): - c.create_search_index(yz['index']) + c.create_search_index(yz['index'], timeout=30000) if yz['btype'] is not None: t = c.bucket_type(yz['btype']) b = t.bucket(yz['bucket']) diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index aaac3f92..2b197966 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -444,7 +444,8 @@ def stream_index(self, bucket, index, startkey, endkey=None, else: raise RiakError('Error streaming secondary index.') - def create_search_index(self, index, schema=None, n_val=None): + def create_search_index(self, index, schema=None, n_val=None, + timeout=None): """ Create a Solr search index for Yokozuna. @@ -454,6 +455,8 @@ def create_search_index(self, index, schema=None, n_val=None): :type schema: string :param n_val: N value of the write :type n_val: int + :param timeout: optional timeout (in ms) + :type timeout: integer, None :rtype boolean """ @@ -468,6 +471,8 @@ def create_search_index(self, index, schema=None, n_val=None): content_dict['schema'] = schema if n_val: content_dict['n_val'] = n_val + if timeout: + content_dict['timeout'] = timeout content = json.dumps(content_dict) # Run the request... diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 5e527c51..e385c698 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -487,7 +487,8 @@ def stream_index(self, bucket, index, startkey, endkey=None, return RiakPbcIndexStream(self, index, return_terms) - def create_search_index(self, index, schema=None, n_val=None): + def create_search_index(self, index, schema=None, n_val=None, + timeout=None): if not self.pb_search_admin(): raise NotImplementedError("Search 2.0 administration is not " "supported for this version") @@ -498,6 +499,8 @@ def create_search_index(self, index, schema=None, n_val=None): if n_val: idx.n_val = n_val req = riak_pb.RpbYokozunaIndexPutReq(index=idx) + if timeout is not None: + req.timeout = timeout self._request(MSG_CODE_YOKOZUNA_INDEX_PUT_REQ, req, MSG_CODE_PUT_RESP) diff --git a/riak/transports/transport.py b/riak/transports/transport.py index be598511..a7428359 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -172,7 +172,8 @@ def get_client_id(self): """ raise NotImplementedError - def create_search_index(self, index, schema=None, n_val=None): + def create_search_index(self, index, schema=None, n_val=None, + timeout=None): """ Creates a yokozuna search index. """ From 6e4f11a4a42b04b87ad0a85cae81b63740af5d54 Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Fri, 16 Oct 2015 03:29:50 +0000 Subject: [PATCH 5/8] Update preflist test --- riak/tests/test_kv.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 4306cc30..d1b28298 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -585,9 +585,10 @@ def test_preflist(self): "bar": "baz"}).store() preflist = bucket.get_preflist(self.key_name) preflist2 = self.client.get_preflist(bucket, self.key_name) + nodes = ['riak@127.0.0.1', 'dev1@127.0.0.1'] for pref in (preflist, preflist2): self.assertEqual(len(pref), 3) - self.assertEqual(pref[0]['node'], 'riak@127.0.0.1') + self.assertIn(pref[0]['node'], nodes) [self.assertTrue(node['primary']) for node in pref] def generate_siblings(self, original, count=5, delay=None): From b01bf1b27d7b9c52dc832aa71c20fe1e4ab723bf Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Fri, 16 Oct 2015 15:33:29 +0000 Subject: [PATCH 6/8] Add in correct permissions for riak_kv.get_preflist --- commands.py | 1 + 1 file changed, 1 insertion(+) diff --git a/commands.py b/commands.py index c2afab0e..06ee3039 100644 --- a/commands.py +++ b/commands.py @@ -238,6 +238,7 @@ class setup_security(Command, security_commands): _grants = { "riak_kv.get": ["any"], + "riak_kv.get_preflist": ["any"], "riak_kv.put": ["any"], "riak_kv.delete": ["any"], "riak_kv.index": ["any"], From 26dfdfcadd8ad1ec8d1c1084f0c6dc8a6e5902f0 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Mon, 19 Oct 2015 14:13:00 -0700 Subject: [PATCH 7/8] Use known working list of ciphers for tests. Tweaks to SecurityError exceptions to fix bug and show more info. --- riak/tests/__init__.py | 5 ++++- riak/tests/test_security.py | 37 +++++++++++++++++++------------ riak/transports/pbc/connection.py | 4 ++-- 3 files changed, 29 insertions(+), 17 deletions(-) diff --git a/riak/tests/__init__.py b/riak/tests/__init__.py index c4d64bf1..d85447ff 100644 --- a/riak/tests/__init__.py +++ b/riak/tests/__init__.py @@ -60,9 +60,12 @@ SECURITY_CERT_PASSWD = os.environ.get('RIAK_TEST_SECURITY_CERT_PASSWD', 'certpass') +SECURITY_CIPHERS = 'DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:AES128-SHA256:AES128-SHA:AES256-SHA256:AES256-SHA:RC4-SHA' + SECURITY_CREDS = None if RUN_SECURITY: SECURITY_CREDS = SecurityCreds(username=SECURITY_USER, password=SECURITY_PASSWD, - cacert_file=SECURITY_CACERT) + cacert_file=SECURITY_CACERT, + ciphers=SECURITY_CIPHERS) SKIP_DATATYPES = int(os.environ.get('SKIP_DATATYPES', '0')) diff --git a/riak/tests/test_security.py b/riak/tests/test_security.py index c76662aa..f0489039 100644 --- a/riak/tests/test_security.py +++ b/riak/tests/test_security.py @@ -20,7 +20,8 @@ import sys from riak.tests import RUN_SECURITY, SECURITY_USER, SECURITY_PASSWD, \ SECURITY_CACERT, SECURITY_KEY, SECURITY_CERT, SECURITY_REVOKED, \ - SECURITY_CERT_USER, SECURITY_CERT_PASSWD, SECURITY_BAD_CERT + SECURITY_CERT_USER, SECURITY_CERT_PASSWD, SECURITY_BAD_CERT, \ + SECURITY_CREDS, SECURITY_CIPHERS from riak.security import SecurityCreds if sys.version_info < (2, 7): unittest = __import__('unittest2') @@ -31,10 +32,7 @@ class SecurityTests(object): @unittest.skipIf(RUN_SECURITY, 'RUN_SECURITY is set') def test_security_disabled(self): - creds = SecurityCreds(username=SECURITY_USER, - password=SECURITY_PASSWD, - cacert_file=SECURITY_CACERT) - client = self.create_client(credentials=creds) + client = self.create_client(credentials=SECURITY_CREDS) myBucket = client.bucket('test') val1 = "foobar" key1 = myBucket.new('x', data=val1) @@ -51,31 +49,39 @@ def test_security_basic_connection(self): @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') def test_security_bad_user(self): - creds = SecurityCreds(username='foo', password=SECURITY_PASSWD, - cacert_file=SECURITY_CACERT) + creds = SecurityCreds(username='foo', + password=SECURITY_PASSWD, + cacert_file=SECURITY_CACERT, + ciphers=SECURITY_CIPHERS) client = self.create_client(credentials=creds) with self.assertRaises(Exception): client.get_buckets() @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') def test_security_bad_password(self): - creds = SecurityCreds(username=SECURITY_USER, password='foo', - cacert_file=SECURITY_CACERT) + creds = SecurityCreds(username=SECURITY_USER, + password='foo', + cacert_file=SECURITY_CACERT, + ciphers=SECURITY_CIPHERS) client = self.create_client(credentials=creds) with self.assertRaises(Exception): client.get_buckets() @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') def test_security_invalid_cert(self): - creds = SecurityCreds(username=SECURITY_USER, password=SECURITY_PASSWD, - cacert_file='/tmp/foo') + creds = SecurityCreds(username=SECURITY_USER, + password=SECURITY_PASSWD, + cacert_file='/tmp/foo', + ciphers=SECURITY_CIPHERS) client = self.create_client(credentials=creds) with self.assertRaises(Exception): client.get_buckets() @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') def test_security_password_without_cacert(self): - creds = SecurityCreds(username=SECURITY_USER, password=SECURITY_PASSWD) + creds = SecurityCreds(username=SECURITY_USER, + password=SECURITY_PASSWD, + ciphers=SECURITY_CIPHERS) client = self.create_client(credentials=creds) with self.assertRaises(Exception): myBucket = client.bucket('test') @@ -87,6 +93,7 @@ def test_security_password_without_cacert(self): def test_security_cert_authentication(self): creds = SecurityCreds(username=SECURITY_CERT_USER, password=SECURITY_CERT_PASSWD, + ciphers=SECURITY_CIPHERS, cert_file=SECURITY_CERT, pkey_file=SECURITY_KEY, cacert_file=SECURITY_CACERT) @@ -107,6 +114,7 @@ def test_security_cert_authentication(self): @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') def test_security_revoked_cert(self): creds = SecurityCreds(username=SECURITY_USER, password=SECURITY_PASSWD, + ciphers=SECURITY_CIPHERS, cacert_file=SECURITY_CACERT, crl_file=SECURITY_REVOKED) # Currently Python >= 2.7.9 and Python 3.x native CRL doesn't seem to @@ -120,6 +128,7 @@ def test_security_revoked_cert(self): @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') def test_security_bad_ca_cert(self): creds = SecurityCreds(username=SECURITY_USER, password=SECURITY_PASSWD, + ciphers=SECURITY_CIPHERS, cacert_file=SECURITY_BAD_CERT) client = self.create_client(credentials=creds) with self.assertRaises(Exception): @@ -128,8 +137,8 @@ def test_security_bad_ca_cert(self): @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') def test_security_ciphers(self): creds = SecurityCreds(username=SECURITY_USER, password=SECURITY_PASSWD, - cacert_file=SECURITY_CACERT, - ciphers='DHE-RSA-AES256-SHA') + ciphers=SECURITY_CIPHERS, + cacert_file=SECURITY_CACERT) client = self.create_client(credentials=creds) myBucket = client.bucket('test') val1 = "foobar" diff --git a/riak/transports/pbc/connection.py b/riak/transports/pbc/connection.py index 293d05c3..0bc58232 100644 --- a/riak/transports/pbc/connection.py +++ b/riak/transports/pbc/connection.py @@ -136,7 +136,7 @@ def _ssl_handshake(self): return True except Exception as e: # fail if *any* exceptions are thrown during SSL handshake - raise SecurityError(e.message) + raise SecurityError(e) else: def _ssl_handshake(self): """ @@ -165,7 +165,7 @@ def _ssl_handshake(self): return True except ssl.SSLError as e: - raise SecurityError(e.library + ": " + e.reason) + raise SecurityError(e) except Exception as e: # fail if *any* exceptions are thrown during SSL handshake raise SecurityError(e) From bdbc35fd2f0c6bd8a3efe0e1376031f112a3f5ae Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Tue, 20 Oct 2015 11:00:48 -0700 Subject: [PATCH 8/8] Some small things to ignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 34e7a5bb..f9515221 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.pyc +.python-version docs/_build @@ -9,6 +10,7 @@ build/ dist/ riak.egg-info/ *.egg +.eggs/ #*# *~