From 312310c2a34ed7160e953950954ec461b251fa0f Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Thu, 11 Feb 2016 14:33:24 -0800 Subject: [PATCH 1/7] Begin migrating to the use of riak-client-tools scripts --- .gitmodules | 3 ++ buildbot/Makefile | 24 ++++++++++----- commands.py | 43 ++++---------------------- riak/tests/__init__.py | 4 +-- riak/tests/test_btypes.py | 15 +++++----- riak/tests/test_datatypes.py | 36 +++++++++++----------- riak/tests/test_mapreduce.py | 58 ++++++++++++++++++------------------ setup.py | 3 +- tools | 1 + 9 files changed, 83 insertions(+), 104 deletions(-) create mode 160000 tools diff --git a/.gitmodules b/.gitmodules index e0cba09c..df75a761 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "riak_pb"] path = riak_pb url = git://github.com/basho/riak_pb.git +[submodule "tools"] + path = tools + url = git://github.com/basho/riak-client-tools.git diff --git a/buildbot/Makefile b/buildbot/Makefile index fd0ee0ed..87d53af1 100644 --- a/buildbot/Makefile +++ b/buildbot/Makefile @@ -2,16 +2,25 @@ ifndef RIAK_DIR $(error RIAK_DIR is not set) endif +PROJDIR = $(realpath $(CURDIR)/..) +TOOLS_DIR = $(PROJDIR)/tools/devrel +CA_DIR = $(PROJDIR)/tools/test-ca RIAK_CONF = $(RIAK_DIR)/etc/riak.conf +ADV_CONF = $(RIAK_DIR)/etc/advanced.config RIAK_ADMIN = $(RIAK_DIR)/bin/riak-admin + +export RIAK_HOST = localhost +export RIAK_PORT = 8087 + CERTS_DIR = $(realpath $(CURDIR))/../riak/tests/resources unexport PYENV_VERSION preconfigure: - @../setup.py preconfigure --riak-conf=$(RIAK_CONF) + $(TOOLS_DIR)/gen-riak-conf $(RIAK_CONF) 8098 8087 18098 $(CA_DIR)/certs/cacert.pem $(CA_DIR)/certs/riak-test-cert.pem $(CA_DIR)/private/riak-test-key.pem + $(TOOLS_DIR)/gen-adv-conf $(ADV_CONF) configure: - @../setup.py configure --riak-admin=$(RIAK_ADMIN) + $(TOOLS_DIR)/riak-cluster-config $(RIAK_ADMIN) 8098 true false configure_timeseries: @../setup.py setup_timeseries --riak-admin=$(RIAK_ADMIN) @@ -26,19 +35,20 @@ lint: @openssl verify -CAfile $(CERTS_DIR)/ca.crt $(CERTS_DIR)/client.crt @openssl verify -CAfile $(CERTS_DIR)/ca.crt $(CERTS_DIR)/server.crt -test: setup test_normal test_security +# TODO test: setup test_normal test_security +test: setup test_normal test_normal: @echo "Testing Riak Python Client (without security)" @../setup.py disable_security --riak-admin=$(RIAK_ADMIN) - @RIAK_TEST_PROTOCOL='pbc' RUN_YZ=1 RUN_DATATYPES=1 RUN_INDEXES=1 ./tox_runner.sh .. - @RIAK_TEST_PROTOCOL='http' RUN_YZ=1 RUN_DATATYPES=1 RUN_INDEXES=1 ./tox_runner.sh .. + @RIAK_TEST_PROTOCOL='pbc' RIAK_TEST_PB_PORT=8087 RUN_YZ=1 RUN_DATATYPES=1 RUN_INDEXES=1 ./tox_runner.sh .. + @RIAK_TEST_PROTOCOL='http' RIAK_TEST_HTTP_PORT=8098 RUN_YZ=1 RUN_DATATYPES=1 RUN_INDEXES=1 ./tox_runner.sh .. test_security: @echo "Testing Riak Python Client (with security)" @../setup.py enable_security --riak-admin=$(RIAK_ADMIN) - @RIAK_TEST_PROTOCOL='pbc' RUN_YZ=1 RUN_INDEXES=1 RUN_SECURITY=1 RUN_POOL=0 RUN_RESOLVE=0 ./tox_runner.sh .. - @RIAK_TEST_PROTOCOL='http' RUN_YZ=1 RUN_INDEXES=1 RUN_SECURITY=1 RUN_POOL=0 RUN_RESOLVE=0 RIAK_TEST_HTTP_PORT=18098 ./tox_runner.sh .. + @RIAK_TEST_PROTOCOL='pbc' RIAK_TEST_PB_PORT=8087 RUN_YZ=1 RUN_INDEXES=1 RUN_SECURITY=1 RUN_POOL=0 RUN_RESOLVE=0 ./tox_runner.sh .. + @RIAK_TEST_PROTOCOL='http' RIAK_TEST_HTTP_PORT=18098 RUN_YZ=1 RUN_INDEXES=1 RUN_SECURITY=1 RUN_POOL=0 RUN_RESOLVE=0 ./tox_runner.sh .. test_timeseries: @echo "Testing Riak Python Client (timeseries)" diff --git a/commands.py b/commands.py index c12b9191..20ce95bc 100644 --- a/commands.py +++ b/commands.py @@ -12,7 +12,7 @@ from subprocess import Popen, PIPE -__all__ = ['create_bucket_types', 'build_messages', +__all__ = ['build_messages', 'setup_security', 'enable_security', 'disable_security', 'setup_timeseries', 'preconfigure', 'configure'] @@ -146,36 +146,6 @@ def _btype_command(self, *args): return cmd -class create_bucket_types(bucket_type_commands, Command): - """ - Creates bucket-types appropriate for testing. By default this will create: - - * `pytest-maps` with ``{"datatype":"map"}`` - * `pytest-sets` with ``{"datatype":"set"}`` - * `pytest-counters` with ``{"datatype":"counter"}`` - * `pytest-consistent` with ``{"consistent":true}`` - * `pytest-write-once` with ``{"write_once": true}`` - * `pytest-mr` - * `pytest` with ``{"allow_mult":false}`` - """ - - description = "create bucket-types used in integration tests" - - user_options = [ - ('riak-admin=', None, 'path to the riak-admin script') - ] - - _props = { - 'pytest-maps': {'datatype': 'map'}, - 'pytest-sets': {'datatype': 'set'}, - 'pytest-counters': {'datatype': 'counter'}, - 'pytest-consistent': {'consistent': True}, - 'pytest-write-once': {'write_once': True}, - 'pytest-mr': {}, - 'pytest': {'allow_mult': False} - } - - class setup_timeseries(bucket_type_commands, Command): """ Creates bucket-types appropriate for timeseries. @@ -477,13 +447,12 @@ class configure(Command): """ Sets up security configuration. - * Run setup_security and create_bucket_types + * Run setup_security """ - description = "create bucket types and security settings for testing" + description = "security settings for testing" - user_options = create_bucket_types.user_options + \ - setup_security.user_options + user_options = setup_security.user_options def initialize_options(self): self.riak_admin = None @@ -491,8 +460,6 @@ def initialize_options(self): self.password = None def finalize_options(self): - bucket = self.distribution.get_command_obj('create_bucket_types') - bucket.riak_admin = self.riak_admin security = self.distribution.get_command_obj('setup_security') security.riak_admin = self.riak_admin security.username = self.username @@ -503,7 +470,7 @@ def run(self): for cmd_name in self.get_sub_commands(): self.run_command(cmd_name) - sub_commands = [('create_bucket_types', None), ('setup_security', None)] + sub_commands = [('setup_security', None)] class ComparableMixin(object): diff --git a/riak/tests/__init__.py b/riak/tests/__init__.py index 29c88619..7ee88648 100644 --- a/riak/tests/__init__.py +++ b/riak/tests/__init__.py @@ -22,10 +22,10 @@ PROTOCOL = os.environ.get('RIAK_TEST_PROTOCOL', 'pbc') PB_HOST = os.environ.get('RIAK_TEST_PB_HOST', HOST) -PB_PORT = int(os.environ.get('RIAK_TEST_PB_PORT', '8087')) +PB_PORT = int(os.environ.get('RIAK_TEST_PB_PORT', '10017')) HTTP_HOST = os.environ.get('RIAK_TEST_HTTP_HOST', HOST) -HTTP_PORT = int(os.environ.get('RIAK_TEST_HTTP_PORT', '8098')) +HTTP_PORT = int(os.environ.get('RIAK_TEST_HTTP_PORT', '10018')) # these ports are used to simulate errors, there shouldn't # be anything listening on either port. diff --git a/riak/tests/test_btypes.py b/riak/tests/test_btypes.py index 3c8b6c1e..30c9d6ac 100644 --- a/riak/tests/test_btypes.py +++ b/riak/tests/test_btypes.py @@ -44,7 +44,7 @@ def test_btype_repr(self): def test_btype_get_props(self): defbtype = self.client.bucket_type("default") - btype = self.client.bucket_type("pytest") + btype = self.client.bucket_type('no_siblings') with self.assertRaises(ValueError): defbtype.get_properties() @@ -55,7 +55,7 @@ def test_btype_get_props(self): def test_btype_set_props(self): defbtype = self.client.bucket_type("default") - btype = self.client.bucket_type("pytest") + btype = self.client.bucket_type('no_siblings') with self.assertRaises(ValueError): defbtype.set_properties({'allow_mult': True}) @@ -72,12 +72,12 @@ def test_btype_set_props(self): btype.set_properties(oldprops) def test_btype_set_props_immutable(self): - btype = self.client.bucket_type("pytest-maps") + btype = self.client.bucket_type("maps") with self.assertRaises(RiakError): btype.set_property('datatype', 'counter') def test_btype_list_buckets(self): - btype = self.client.bucket_type("pytest") + btype = self.client.bucket_type('no_siblings') bucket = btype.bucket(self.bucket_name) obj = bucket.new(self.key_name) obj.data = [1, 2, 3] @@ -91,7 +91,7 @@ def test_btype_list_buckets(self): self.assertIn(bucket, buckets) def test_btype_list_keys(self): - btype = self.client.bucket_type("pytest") + btype = self.client.bucket_type('no_siblings') bucket = btype.bucket(self.bucket_name) obj = bucket.new(self.key_name) @@ -140,7 +140,7 @@ def test_default_btype_list_keys(self): self.assertItemsEqual(keys, oldapikeys) def test_multiget_bucket_types(self): - btype = self.client.bucket_type('pytest') + btype = self.client.bucket_type('no_siblings') bucket = btype.bucket(self.bucket_name) for i in range(100): @@ -155,8 +155,7 @@ def test_multiget_bucket_types(self): self.assertEqual(btype, mobj.bucket.bucket_type) def test_write_once_bucket_type(self): - btype = self.client.bucket_type('pytest-write-once') - btype.set_property('write_once', True) + btype = self.client.bucket_type('write_once') bucket = btype.bucket(self.bucket_name) for i in range(100): diff --git a/riak/tests/test_datatypes.py b/riak/tests/test_datatypes.py index c832583b..39166069 100644 --- a/riak/tests/test_datatypes.py +++ b/riak/tests/test_datatypes.py @@ -157,7 +157,7 @@ class DatatypeIntegrationTests(IntegrationTestBase, unittest.TestCase, Comparison): def test_dt_counter(self): - btype = self.client.bucket_type('pytest-counters') + btype = self.client.bucket_type('counters') bucket = btype.bucket(self.bucket_name) mycount = datatypes.Counter(bucket, self.key_name) mycount.increment(5) @@ -173,7 +173,7 @@ def test_dt_counter(self): self.assertEqual(2, mycount.value) def test_dt_set(self): - btype = self.client.bucket_type('pytest-sets') + btype = self.client.bucket_type('sets') bucket = btype.bucket(self.bucket_name) myset = datatypes.Set(bucket, self.key_name) myset.add('Sean') @@ -195,7 +195,7 @@ def test_dt_set(self): self.assertNotIn('Sean', myset) def test_dt_map(self): - btype = self.client.bucket_type('pytest-maps') + btype = self.client.bucket_type('maps') bucket = btype.bucket(self.bucket_name) mymap = datatypes.Map(bucket, self.key_name) @@ -230,7 +230,7 @@ def test_dt_map(self): self.assertItemsEqual(['thing1', 'thing2'], mymap.sets['f'].value) def test_dt_set_remove_without_context(self): - btype = self.client.bucket_type('pytest-sets') + btype = self.client.bucket_type('sets') bucket = btype.bucket(self.bucket_name) set = datatypes.Set(bucket, self.key_name) @@ -241,7 +241,7 @@ def test_dt_set_remove_without_context(self): set.discard("Y") def test_dt_set_remove_fetching_context(self): - btype = self.client.bucket_type('pytest-sets') + btype = self.client.bucket_type('sets') bucket = btype.bucket(self.bucket_name) set = datatypes.Set(bucket, self.key_name) @@ -257,7 +257,7 @@ def test_dt_set_remove_fetching_context(self): self.assertItemsEqual(['X', 'Y'], set2.value) def test_dt_set_add_twice(self): - btype = self.client.bucket_type('pytest-sets') + btype = self.client.bucket_type('sets') bucket = btype.bucket(self.bucket_name) set = datatypes.Set(bucket, self.key_name) @@ -273,7 +273,7 @@ def test_dt_set_add_twice(self): self.assertItemsEqual(['X', 'Y'], set2.value) def test_dt_set_add_wins_in_same_op(self): - btype = self.client.bucket_type('pytest-sets') + btype = self.client.bucket_type('sets') bucket = btype.bucket(self.bucket_name) set = datatypes.Set(bucket, self.key_name) @@ -290,7 +290,7 @@ def test_dt_set_add_wins_in_same_op(self): self.assertItemsEqual(['X', 'Y'], set2.value) def test_dt_set_add_wins_in_same_op_reversed(self): - btype = self.client.bucket_type('pytest-sets') + btype = self.client.bucket_type('sets') bucket = btype.bucket(self.bucket_name) set = datatypes.Set(bucket, self.key_name) @@ -307,7 +307,7 @@ def test_dt_set_add_wins_in_same_op_reversed(self): self.assertItemsEqual(['X', 'Y'], set2.value) def test_dt_set_remove_old_context(self): - btype = self.client.bucket_type('pytest-sets') + btype = self.client.bucket_type('sets') bucket = btype.bucket(self.bucket_name) set = datatypes.Set(bucket, self.key_name) @@ -328,7 +328,7 @@ def test_dt_set_remove_old_context(self): self.assertItemsEqual(['X', 'Y', 'Z'], set2.value) def test_dt_set_remove_updated_context(self): - btype = self.client.bucket_type('pytest-sets') + btype = self.client.bucket_type('sets') bucket = btype.bucket(self.bucket_name) set = datatypes.Set(bucket, self.key_name) @@ -348,7 +348,7 @@ def test_dt_set_remove_updated_context(self): self.assertItemsEqual(['X', 'Y'], set2.value) def test_dt_map_remove_set_update_same_op(self): - btype = self.client.bucket_type('pytest-maps') + btype = self.client.bucket_type('maps') bucket = btype.bucket(self.bucket_name) map = datatypes.Map(bucket, self.key_name) @@ -365,7 +365,7 @@ def test_dt_map_remove_set_update_same_op(self): self.assertItemsEqual(["Z"], map2.sets['set']) def test_dt_map_remove_counter_increment_same_op(self): - btype = self.client.bucket_type('pytest-maps') + btype = self.client.bucket_type('maps') bucket = btype.bucket(self.bucket_name) map = datatypes.Map(bucket, self.key_name) @@ -382,7 +382,7 @@ def test_dt_map_remove_counter_increment_same_op(self): self.assertEqual(2, map2.counters['counter'].value) def test_dt_map_remove_map_update_same_op(self): - btype = self.client.bucket_type('pytest-maps') + btype = self.client.bucket_type('maps') bucket = btype.bucket(self.bucket_name) map = datatypes.Map(bucket, self.key_name) @@ -399,7 +399,7 @@ def test_dt_map_remove_map_update_same_op(self): self.assertItemsEqual(["Z"], map2.maps['map'].sets['set']) def test_dt_set_return_body_true_default(self): - btype = self.client.bucket_type('pytest-sets') + btype = self.client.bucket_type('sets') bucket = btype.bucket(self.bucket_name) myset = bucket.new(self.key_name) myset.add('X') @@ -416,7 +416,7 @@ def test_dt_set_return_body_true_default(self): self.assertItemsEqual(myset.value, ['Y']) def test_dt_map_return_body_true_default(self): - btype = self.client.bucket_type('pytest-maps') + btype = self.client.bucket_type('maps') bucket = btype.bucket(self.bucket_name) mymap = bucket.new(self.key_name) mymap.sets['a'].add('X') @@ -440,19 +440,19 @@ def test_dt_map_return_body_true_default(self): self.assertEqual(mymap.value, {}) def test_delete_datatype(self): - ctype = self.client.bucket_type('pytest-counters') + ctype = self.client.bucket_type('counters') cbucket = ctype.bucket(self.bucket_name) counter = cbucket.new(self.key_name) counter.increment(5) counter.store() - stype = self.client.bucket_type('pytest-sets') + stype = self.client.bucket_type('sets') sbucket = stype.bucket(self.bucket_name) set_ = sbucket.new(self.key_name) set_.add("Brett") set_.store() - mtype = self.client.bucket_type('pytest-maps') + mtype = self.client.bucket_type('maps') mbucket = mtype.bucket(self.bucket_name) map_ = mbucket.new(self.key_name) map_.sets['people'].add('Sean') diff --git a/riak/tests/test_mapreduce.py b/riak/tests/test_mapreduce.py index b6cd068f..a1827398 100644 --- a/riak/tests/test_mapreduce.py +++ b/riak/tests/test_mapreduce.py @@ -18,7 +18,7 @@ import unittest -testrun_yz_mr = {'btype': 'pytest-mr', +testrun_yz_mr = {'btype': 'mr', 'bucket': 'mrbucket', 'index': 'mrbucket'} @@ -116,16 +116,16 @@ def test_erlang_map_reduce(self): def test_erlang_map_reduce_bucket_type(self): # Create the object... - btype = self.client.bucket_type("pytest") + btype = self.client.bucket_type('no_siblings') bucket = btype.bucket(self.bucket_name) bucket.new("foo", 2).store() bucket.new("bar", 2).store() bucket.new("baz", 4).store() # Run the map... result = self.client \ - .add(self.bucket_name, "foo", bucket_type="pytest") \ - .add(self.bucket_name, "bar", bucket_type="pytest") \ - .add(self.bucket_name, "baz", bucket_type="pytest") \ + .add(self.bucket_name, "foo", bucket_type='no_siblings') \ + .add(self.bucket_name, "bar", bucket_type='no_siblings') \ + .add(self.bucket_name, "baz", bucket_type='no_siblings') \ .map(["riak_kv_mapreduce", "map_object_value"]) \ .reduce(["riak_kv_mapreduce", "reduce_set_union"]) \ .run() @@ -159,7 +159,7 @@ def test_erlang_source_map_reduce(self): def test_erlang_source_map_reduce_bucket_type(self): # Create the object... - btype = self.client.bucket_type("pytest") + btype = self.client.bucket_type('no_siblings') bucket = btype.bucket(self.bucket_name) bucket.new("foo", 2).store() bucket.new("bar", 3).store() @@ -168,9 +168,9 @@ def test_erlang_source_map_reduce_bucket_type(self): # Run the map... try: result = self.client \ - .add(self.bucket_name, "foo", bucket_type="pytest") \ - .add(self.bucket_name, "bar", bucket_type="pytest") \ - .add(self.bucket_name, "baz", bucket_type="pytest") \ + .add(self.bucket_name, "foo", bucket_type='no_siblings') \ + .add(self.bucket_name, "bar", bucket_type='no_siblings') \ + .add(self.bucket_name, "baz", bucket_type='no_siblings') \ .map("""fun(Object, _KD, _A) -> Value = riak_object:get_value(Object), [Value] @@ -255,12 +255,12 @@ def test_javascript_named_map(self): def test_javascript_named_map_bucket_type(self): # Create the object... - btype = self.client.bucket_type("pytest") + btype = self.client.bucket_type('no_siblings') bucket = btype.bucket(self.bucket_name) bucket.new("foo", 2).store() # Run the map... result = self.client \ - .add(self.bucket_name, "foo", bucket_type="pytest") \ + .add(self.bucket_name, "foo", bucket_type='no_siblings') \ .map("Riak.mapValuesJson") \ .run() self.assertEqual(result, [2]) @@ -283,16 +283,16 @@ def test_javascript_source_map_reduce(self): def test_javascript_source_map_reduce_bucket_type(self): # Create the object... - btype = self.client.bucket_type("pytest") + btype = self.client.bucket_type('no_siblings') bucket = btype.bucket(self.bucket_name) bucket.new("foo", 2).store() bucket.new("bar", 3).store() bucket.new("baz", 4).store() # Run the map... result = self.client \ - .add(self.bucket_name, "foo", bucket_type="pytest") \ - .add(self.bucket_name, "bar", bucket_type="pytest") \ - .add(self.bucket_name, "baz", bucket_type="pytest") \ + .add(self.bucket_name, "foo", bucket_type='no_siblings') \ + .add(self.bucket_name, "bar", bucket_type='no_siblings') \ + .add(self.bucket_name, "baz", bucket_type='no_siblings') \ .map("function (v) { return [1]; }") \ .reduce("Riak.reduceSum") \ .run() @@ -316,16 +316,16 @@ def test_javascript_named_map_reduce(self): def test_javascript_named_map_reduce_bucket_type(self): # Create the object... - btype = self.client.bucket_type("pytest") + btype = self.client.bucket_type('no_siblings') bucket = btype.bucket(self.bucket_name) bucket.new("foo", 2).store() bucket.new("bar", 3).store() bucket.new("baz", 4).store() # Run the map... result = self.client \ - .add(self.bucket_name, "foo", bucket_type="pytest") \ - .add(self.bucket_name, "bar", bucket_type="pytest") \ - .add(self.bucket_name, "baz", bucket_type="pytest") \ + .add(self.bucket_name, "foo", bucket_type='no_siblings') \ + .add(self.bucket_name, "bar", bucket_type='no_siblings') \ + .add(self.bucket_name, "baz", bucket_type='no_siblings') \ .map("Riak.mapValuesJson") \ .reduce("Riak.reduceSum") \ .run() @@ -347,14 +347,14 @@ def test_javascript_bucket_map_reduce(self): def test_javascript_bucket_map_reduceP_bucket_type(self): # Create the object... - btype = self.client.bucket_type("pytest") + btype = self.client.bucket_type('no_siblings') bucket = btype.bucket("bucket_%s" % self.randint()) bucket.new("foo", 2).store() bucket.new("bar", 3).store() bucket.new("baz", 4).store() # Run the map... result = self.client \ - .add(bucket.name, bucket_type="pytest") \ + .add(bucket.name, bucket_type='no_siblings') \ .map("Riak.mapValuesJson") \ .reduce("Riak.reduceSum") \ .run() @@ -378,16 +378,16 @@ def test_javascript_arg_map_reduce(self): def test_javascript_arg_map_reduce_bucket_type(self): # Create the object... - btype = self.client.bucket_type("pytest") + btype = self.client.bucket_type('no_siblings') bucket = btype.bucket(self.bucket_name) bucket.new("foo", 2).store() # Run the map... result = self.client \ - .add(self.bucket_name, "foo", 5, bucket_type="pytest") \ - .add(self.bucket_name, "foo", 10, bucket_type="pytest") \ - .add(self.bucket_name, "foo", 15, bucket_type="pytest") \ - .add(self.bucket_name, "foo", -15, bucket_type="pytest") \ - .add(self.bucket_name, "foo", -5, bucket_type="pytest") \ + .add(self.bucket_name, "foo", 5, bucket_type='no_siblings') \ + .add(self.bucket_name, "foo", 10, bucket_type='no_siblings') \ + .add(self.bucket_name, "foo", 15, bucket_type='no_siblings') \ + .add(self.bucket_name, "foo", -15, bucket_type='no_siblings') \ + .add(self.bucket_name, "foo", -5, bucket_type='no_siblings') \ .map("function(v, arg) { return [arg]; }") \ .reduce("Riak.reduceSum") \ .run() @@ -409,14 +409,14 @@ def test_key_filters(self): self.assertEqual(result, ["yahoo-20090613"]) def test_key_filters_bucket_type(self): - btype = self.client.bucket_type("pytest") + btype = self.client.bucket_type('no_siblings') bucket = btype.bucket("kftest") bucket.new("basho-20101215", 1).store() bucket.new("google-20110103", 2).store() bucket.new("yahoo-20090613", 3).store() result = self.client \ - .add("kftest", bucket_type="pytest") \ + .add("kftest", bucket_type='no_siblings') \ .add_key_filters([["tokenize", "-", 2]]) \ .add_key_filter("ends_with", "0613") \ .map("function (v, keydata) { return [v.key]; }") \ diff --git a/setup.py b/setup.py index 5219948d..1c40976b 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ import platform from setuptools import setup, find_packages from version import get_version -from commands import preconfigure, configure, create_bucket_types, \ +from commands import preconfigure, configure, \ setup_security, enable_security, disable_security, setup_timeseries, \ build_messages @@ -46,7 +46,6 @@ cmdclass={ 'build_messages': build_messages, 'setup_timeseries': setup_timeseries, - 'create_bucket_types': create_bucket_types, 'setup_security': setup_security, 'preconfigure': preconfigure, 'configure': configure, diff --git a/tools b/tools new file mode 160000 index 00000000..343c4a48 --- /dev/null +++ b/tools @@ -0,0 +1 @@ +Subproject commit 343c4a485e26bcee971a41faf9234d65a3fb8b53 From bbe7009d5d336692518c66c802d7ac4ebb710d30 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Thu, 11 Feb 2016 14:48:10 -0800 Subject: [PATCH 2/7] Convert to md format --- README.md | 245 +++++++++++++++++++++++++++++++++++++++++++ README.rst | 298 ----------------------------------------------------- 2 files changed, 245 insertions(+), 298 deletions(-) create mode 100644 README.md delete mode 100644 README.rst diff --git a/README.md b/README.md new file mode 100644 index 00000000..fc921135 --- /dev/null +++ b/README.md @@ -0,0 +1,245 @@ +Python Client for Riak +====================== + +Documentation +============= + +[Documentation for the Riak Python Client Library](http://basho.github.io/riak-python-client/index.html) is available here. The documentation source is found in [`docs/` subdirectory](https://github.com/basho/riak-python-client/tree/master/docs) and can be built with [Sphinx](http://sphinx.pocoo.org/). + +Documentation for Riak is available [here](http://docs.basho.com/riak/latest). + +Install +======= + +The recommended versions of Python for use with this client are Python `2.7.x`, `3.3.x`, `3.4.x` and `3.5.x`. The latest version from each series should be preferred. + +Riak TS (Timeseries) +=================== + +You must use version `2.7.11`, `3.4.4` or `3.5.1` (or greater within a version series). Otherwise you will be affected by [this Python bug](https://bugs.python.org/issue23517). + +From Source +----------- + +``sh +python setup.py install +`` + +There are additional dependencies on Python packages `setuptools` and `protobuf`. + +From PyPI +--------- + +Official packages are signed and published to [PyPI](https://pypi.python.org/pypi/riak). + +To install from [PyPI](https://pypi.python.org/pypi/riak) directly you can use `pip`. + +``sh +pip install riak +`` + +Testing +======= + +To setup the default test configuration build a test Riak node (from a `riak` directory) + +``sh +make rel +`` + +See [Basic Cluster Setup](http://docs.basho.com/riak/2.0.0/ops/building/basic-cluster-setup/) for more details. + +For all of the simple default values, set the `RIAK_DIR` environment variable to the root of your Riak installation. Then from the `riak-python-client` directory + +``sh +make -C buildbot preconfigure +`` + +Start your Riak node with `riak start` from the the Riak directory, then + +``sh +make -C buildbot configure +make -C buildbot test +`` + +That will run the test suite twice: once with security enabled and once without. + +Testing Options +--------------- + +If you wish to change the default options you can run the setup by hand. First configure the test node by adjusting the `riak.conf` settings, where `RIAK_DIR` is the path to the top your Riak installation + +```sh +python setup.py preconfigure --riak-conf=$RIAK_DIR/etc/riak.conf +``` + +Optionally the hostname and port numbers can be changed, too, via these arguments: + +* `--host=` IP of host running Riak (default is `localhost`) +* `--pb-port=` protocol buffers port number (default is `8087`) +* `--http-port=` http port number (default is `8098`) +* `--https-port=` https port number (default is `8099`) + +You may alternately add these lines to `setup.cfg` + +```ini +[preconfigure] +riak-conf=/Users/sean/dev/riak/rel/riak/etc/riak.conf +host=localhost +pb-port=8087 +http-port=8098 +https-port=8099 +``` + +Next start the test node. Once it is running, a test configuration is installed which includes security test users and bucket types + +```sh +python setup.py configure --riak-admin=$RIAK_DIR/bin/riak-admin +``` + +Optionally these configuration settings can be changed, too: + +* `--username=` test user account (default is `testuser`) +* `--password=` password for test user account (default is `testpassword`) +* `--certuser=` secruity test user account (default is `certuser`) +* `--certpass=` password for security test user account (default is `certpass`) + +Similarly `setup.cfg` may be modified instead. To run the tests against a Riak server (with configured TCP port configuration) on localhost, execute + +```sh +python setup.py test +``` + +Connections to Riak in Tests +---------------------------- + +If your Riak server isn't running on localhost or you have built a Riak devrel from source, use the environment variables `RIAK_TEST_HOST`, `RIAK_TEST_HTTP_PORT` and `RIAK_TEST_PB_PORT` to specify where to find the Riak server. `RIAK_TEST_PROTOCOL` to specify which protocol to test. Can be either `pbc` or `http`. + +Some of the connection tests need port numbers that are NOT in use. If ports 1023 and 1022 are in use on your test system, set the environment variables `DUMMY_HTTP_PORT` and `DUMMY_PB_PORT` to unused port numbers. + +Testing Search +-------------- + +If you don't have [Riak Search](http://docs.basho.com/riak/latest/dev/using/search/) enabled, you can set the `RUN_SEARCH` environment variable to 0 skip those tests. + +If you don't have [Search 2.0](https://github.com/basho/yokozuna) enabled, you can set the `RUN_YZ` environment variable to 0 to skip those tests. + +Testing Bucket Types (Riak 2+) +------------------------------ + +To test bucket-types, you must run the `create_bucket_types` setup command, which will create the bucket-types used in testing, or create them manually yourself. It can be run like so (substituting `$RIAK` with the root of your Riak install) + +```sh +./setup.py create_bucket_types --riak-admin=$RIAK/bin/riak-admin +``` + +You may alternately add these lines to `setup.cfg` + +```ini +[create_bucket_types] +riak-admin=/Users/sean/dev/riak/rel/riak/bin/riak-admin +``` + +To skip the bucket-type tests, set the `RUN_BTYPES` environment variable to `0`. + +Testing Data Types (Riak 2+) +---------------------------- + +To test data types, you must set up bucket types (see above.) + +To skip the data type tests, set the `RUN_DATATYPES` environment variable to `0`. + +Testing Timeseries (Riak 2.1+) +------------------------------ + +To test timeseries data, you must run the `setup_timeseries` command, which will create the bucket-types used in testing, or create them manually yourself. It can be run like so (substituting `$RIAK` with the root of your Riak install) + +```sh +./setup.py setup_timeseries --riak-admin=$RIAK/bin/riak-admin +``` + +You may alternately add these lines to `setup.cfg` + +```sh +[setup_timeseries] +riak-admin=/Users/sean/dev/riak/rel/riak/bin/riak-admin +``` + +To enable the timeseries tests, set the `RUN_TIMESERIES` environment variable to `1`. + +Testing Secondary Indexes +------------------------- + +To test [Secondary Indexes](http://docs.basho.com/riak/2.0.0/dev/using/2i/), the `RUN_INDEXES` environment variable must be set to 1 (or 0 to skip them.) + +Testing Security (Riak 2+) +-------------------------- + +By default [Security](http://docs.basho.com/riak/2.0.0beta1/ops/running/authz/) is not enabled on Riak. Once `security = on` is configured in the `riak.conf` file it can be enabled with `riak-admin`. + +If you have set up the test environment outlined in the Testing section you can go ahead and use this command to enable security + +```sh +python setup.py enable_security --riak-admin=$RIAK_DIR/bin/riak-admin +``` + +Once you are done testing security you can also + +```sh +python setup.py disable_security --riak-admin=$RIAK_DIR/bin/riak-admin +``` + +To run the tests + +```sh +RUN_SECURITY=1 RIAK_TEST_HTTP_PORT=18098 python setup.py test +``` + +Contributors +-------------------------- + +* Andrew Thompson +* Andy Gross +* Armon Dadgar +* Brett Hazen +* Brett Hoerner +* Brian Roach +* Bryan Fink +* Daniel Lindsley +* Daniel Néri +* Daniel Reverri +* David Koblas +* Dmitry Rozhkov +* Eric Florenzano +* Eric Moritz +* Filip de Waard +* Gilles Devaux +* Greg Nelson +* Gregory Burd +* Greg Stein +* Ian Plosker +* Jayson Baird +* Jeffrey Massung +* Jon Meredith +* Josip Lisec +* Justin Sheehy +* Kevin Smith +* [Luke Bakken](https://github.com/lukebakken) +* Mark Erdmann +* Mark Phillips +* Mathias Meyer +* Matt Heitzenroder +* Mikhail Sobolev +* Reid Draper +* Russell Brown +* Rusty Klophaus +* Rusty Klophaus +* Scott Lystig Fritchie +* Sean Cribbs +* Shuhao Wu +* Silas Sewell +* Socrates Lee +* Soren Hansen +* Sreejith Kesavan +* Timothée Peignier +* William Kral diff --git a/README.rst b/README.rst deleted file mode 100644 index 6e9b66ba..00000000 --- a/README.rst +++ /dev/null @@ -1,298 +0,0 @@ -====================== -Python Client for Riak -====================== - -Documentation -============= - -`Documentation for the Riak Python Client Library -`_ is available -here. The documentation source is found in `docs/ subdirectory -`_ and -can be built with `Sphinx `_. - -Documentation for Riak is available at http://docs.basho.com/riak/latest - -Install -======= - -The recommended versions of Python for use with this client are Python -`2.7.x`, `3.3.x`, `3.4.x` and `3.5.x`. The latest version from each series -should be preferred. - -Riak TS (Timeseries) -=================== - -You must use version `2.7.11`, `3.4.4` or `3.5.1` (or greater within a version series). -Otherwise you will be affected by `this Python bug `_. - - -From Source ------------ - -.. code-block:: console - - python setup.py install - -There are additional dependencies on Python packages `setuptools` and `protobuf`. - -From PyPI ---------- - -Official packages are signed and published to `PyPI -`_. - -To install from `PyPI `_ directly you can use -`pip`. - -.. code-block:: console - - pip install riak - - -Testing -======= - -To setup the default test configuration build a test Riak node (from -a ``riak`` directory) - -.. code-block:: console - - make rel - -See `Basic Cluster Setup -`_ -for more details. - -For all of the simple default values, set the ``RIAK_DIR`` environment -variable to the root of your Riak installation. Then from the -``riak-python-client`` directory - -.. code-block:: console - - make -C buildbot preconfigure - -Start your Riak node with ``riak start`` from the the Riak directory, then - -.. code-block:: console - - make -C buildbot configure - make -C buildbot test - -That will run the test suite twice: once with security enabled and once -without. - -Testing Options ---------------- - -If you wish to change the default options you can run the setup by hand. -First configure the test node by adjusting the ``riak.conf`` -settings, where ``RIAK_DIR`` is the path to the top your -Riak installation - -.. code-block:: console - - python setup.py preconfigure --riak-conf=$RIAK_DIR/etc/riak.conf - -Optionally the hostname and port numbers can be changed, too, via these -arguments: - - - ``--host=`` IP of host running Riak (default is ``localhost``) - - ``--pb-port=`` protocol buffers port number (default is ``8087``) - - ``--http-port=`` http port number (default is ``8098``) - - ``--https-port=`` https port number (default is ``8099``) - -You may alternately add these lines to ``setup.cfg`` - -.. code-block:: ini - - [preconfigure] - riak-conf=/Users/sean/dev/riak/rel/riak/etc/riak.conf - host=localhost - pb-port=8087 - http-port=8098 - https-port=8099 - -Next start the test node. Once it is running, a test configuration is -installed which includes security test users and bucket types - -.. code-block:: console - - python setup.py configure --riak-admin=$RIAK_DIR/bin/riak-admin - -Optionally these configuration settings can be changed, too: - - - ``--username=`` test user account (default is ``testuser``) - - ``--password=`` password for test user account (default is - ``testpassword``) - - ``--certuser=`` secruity test user account (default is ``certuser``) - - ``--certpass=`` password for security test user account (default is - ``certpass``) - -Similarly ``setup.cfg`` may be modified instead. To run the tests against a -Riak server (with configured TCP port configuration) on localhost, execute - -.. code-block:: console - - python setup.py test - -Connections to Riak in Tests ----------------------------- - -If your Riak server isn't running on localhost or you have built a -Riak devrel from source, use the environment variables -``RIAK_TEST_HOST``, ``RIAK_TEST_HTTP_PORT`` and -``RIAK_TEST_PB_PORT`` to specify where to find the Riak server. -``RIAK_TEST_PROTOCOL`` to specify which protocol to test. Can be -either ``pbc`` or ``http``. - -Some of the connection tests need port numbers that are NOT in use. If -ports 1023 and 1022 are in use on your test system, set the -environment variables ``DUMMY_HTTP_PORT`` and ``DUMMY_PB_PORT`` to -unused port numbers. - -Testing Search --------------- - -If you don't have `Riak Search -`_ enabled, you -can set the ``RUN_SEARCH`` environment variable to 0 skip those -tests. - -If you don't have `Search 2.0 `_ -enabled, you can set the ``RUN_YZ`` environment variable to 0 to skip -those tests. - -Testing Bucket Types (Riak 2+) ------------------------------- - -To test bucket-types, you must run the ``create_bucket_types`` setup -command, which will create the bucket-types used in testing, or create -them manually yourself. It can be run like so (substituting ``$RIAK`` -with the root of your Riak install) - -.. code-block:: console - - ./setup.py create_bucket_types --riak-admin=$RIAK/bin/riak-admin - -You may alternately add these lines to `setup.cfg` - -.. code-block:: ini - - [create_bucket_types] - riak-admin=/Users/sean/dev/riak/rel/riak/bin/riak-admin - -To skip the bucket-type tests, set the ``RUN_BTYPES`` environment -variable to ``0``. - -Testing Data Types (Riak 2+) ----------------------------- - -To test data types, you must set up bucket types (see above.) - -To skip the data type tests, set the ``RUN_DATATYPES`` environment -variable to ``0``. - -Testing Timeseries (Riak 2.1+) ------------------------------- - -To test timeseries data, you must run the ``setup_timeseries`` command, -which will create the bucket-types used in testing, or create them -manually yourself. It can be run like so (substituting ``$RIAK`` with -the root of your Riak install) - -.. code-block:: console - - ./setup.py setup_timeseries --riak-admin=$RIAK/bin/riak-admin - -You may alternately add these lines to `setup.cfg` - -.. code-block:: ini - - [setup_timeseries] - riak-admin=/Users/sean/dev/riak/rel/riak/bin/riak-admin - -To enable the timeseries tests, set the ``RUN_TIMESERIES`` environment -variable to ``1``. - -Testing Secondary Indexes -------------------------- - -To test -`Secondary Indexes `_, -the ``RUN_INDEXES`` environment variable must be set to 1 (or 0 to skip them.) - -Testing Security (Riak 2+) --------------------------- - -By default -`Security `_ is not -enabled on Riak. Once ``security = on`` is configured in the ``riak.conf`` -file it can be enabled with ``riak-admin``. - -If you have set up the test environment outlined in the `Testing`_ section -you can go ahead and use this command to enable security - -.. code-block:: console - - python setup.py enable_security --riak-admin=$RIAK_DIR/bin/riak-admin - -Once you are done testing security you can also - -.. code-block:: console - - python setup.py disable_security --riak-admin=$RIAK_DIR/bin/riak-admin - -To run the tests, then simply - -.. code-block:: console - - RUN_SECURITY=1 RIAK_TEST_HTTP_PORT=18098 python setup.py test - -Contributors --------------------------- - - Andrew Thompson - - Andy Gross - - Armon Dadgar - - Brett Hazen - - Brett Hoerner - - Brian Roach - - Bryan Fink - - Daniel Lindsley - - Daniel Néri - - Daniel Reverri - - David Koblas - - Dmitry Rozhkov - - Eric Florenzano - - Eric Moritz - - Filip de Waard - - Gilles Devaux - - Greg Nelson - - Greg Stein - - Gregory Burd - - Ian Plosker - - Jayson Baird - - Jeffrey Massung - - Jon Meredith - - Josip Lisec - - Justin Sheehy - - Kevin Smith - - `Luke Bakken `_ - - Mark Erdmann - - Mark Phillips - - Mathias Meyer - - Matt Heitzenroder - - Mikhail Sobolev - - Reid Draper - - Russell Brown - - Rusty Klophaus - - Rusty Klophaus - - Scott Lystig Fritchie - - Sean Cribbs - - Shuhao Wu - - Silas Sewell - - Socrates Lee - - Soren Hansen - - Sreejith Kesavan - - Timothée Peignier - - William Kral From d537563abf4479ed03285caac4d61ba1a567d810 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Thu, 11 Feb 2016 15:15:31 -0800 Subject: [PATCH 3/7] Fix code blocks --- README.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index fc921135..52f41f1e 100644 --- a/README.md +++ b/README.md @@ -21,9 +21,9 @@ You must use version `2.7.11`, `3.4.4` or `3.5.1` (or greater within a version s From Source ----------- -``sh +```sh python setup.py install -`` +``` There are additional dependencies on Python packages `setuptools` and `protobuf`. @@ -34,33 +34,33 @@ Official packages are signed and published to [PyPI](https://pypi.python.org/pyp To install from [PyPI](https://pypi.python.org/pypi/riak) directly you can use `pip`. -``sh +```sh pip install riak -`` +``` Testing ======= To setup the default test configuration build a test Riak node (from a `riak` directory) -``sh +```sh make rel -`` +``` See [Basic Cluster Setup](http://docs.basho.com/riak/2.0.0/ops/building/basic-cluster-setup/) for more details. For all of the simple default values, set the `RIAK_DIR` environment variable to the root of your Riak installation. Then from the `riak-python-client` directory -``sh +```sh make -C buildbot preconfigure -`` +``` Start your Riak node with `riak start` from the the Riak directory, then -``sh +```sh make -C buildbot configure make -C buildbot test -`` +``` That will run the test suite twice: once with security enabled and once without. From 00e27cb6c76c026373785b5ba2dc0ce94f618899 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Sat, 13 Feb 2016 18:14:22 -0800 Subject: [PATCH 4/7] Migrate more stuff to riak-client-tools --- buildbot/Makefile | 21 +-- commands.py | 304 +----------------------------- riak/tests/__init__.py | 43 +++-- riak/tests/test_kv.py | 2 +- riak/tests/test_security.py | 4 +- riak/transports/http/__init__.py | 3 +- riak/transports/pbc/connection.py | 7 +- riak/transports/security.py | 1 + setup.py | 11 +- tools | 2 +- 10 files changed, 51 insertions(+), 347 deletions(-) diff --git a/buildbot/Makefile b/buildbot/Makefile index 87d53af1..8272fdf1 100644 --- a/buildbot/Makefile +++ b/buildbot/Makefile @@ -2,6 +2,8 @@ ifndef RIAK_DIR $(error RIAK_DIR is not set) endif +unexport PYENV_VERSION + PROJDIR = $(realpath $(CURDIR)/..) TOOLS_DIR = $(PROJDIR)/tools/devrel CA_DIR = $(PROJDIR)/tools/test-ca @@ -9,18 +11,12 @@ RIAK_CONF = $(RIAK_DIR)/etc/riak.conf ADV_CONF = $(RIAK_DIR)/etc/advanced.config RIAK_ADMIN = $(RIAK_DIR)/bin/riak-admin -export RIAK_HOST = localhost -export RIAK_PORT = 8087 - -CERTS_DIR = $(realpath $(CURDIR))/../riak/tests/resources -unexport PYENV_VERSION - preconfigure: $(TOOLS_DIR)/gen-riak-conf $(RIAK_CONF) 8098 8087 18098 $(CA_DIR)/certs/cacert.pem $(CA_DIR)/certs/riak-test-cert.pem $(CA_DIR)/private/riak-test-key.pem $(TOOLS_DIR)/gen-adv-conf $(ADV_CONF) configure: - $(TOOLS_DIR)/riak-cluster-config $(RIAK_ADMIN) 8098 true false + $(TOOLS_DIR)/riak-cluster-config $(RIAK_ADMIN) 8098 true true configure_timeseries: @../setup.py setup_timeseries --riak-admin=$(RIAK_ADMIN) @@ -32,27 +28,24 @@ lint: @pip install --upgrade pep8 flake8 @cd ..; pep8 --exclude=riak/pb riak *.py @cd ..; flake8 --exclude=riak/pb riak *.py - @openssl verify -CAfile $(CERTS_DIR)/ca.crt $(CERTS_DIR)/client.crt - @openssl verify -CAfile $(CERTS_DIR)/ca.crt $(CERTS_DIR)/server.crt -# TODO test: setup test_normal test_security -test: setup test_normal +test: setup test_normal test_security test_normal: @echo "Testing Riak Python Client (without security)" - @../setup.py disable_security --riak-admin=$(RIAK_ADMIN) + @$(RIAK_ADMIN) security disable @RIAK_TEST_PROTOCOL='pbc' RIAK_TEST_PB_PORT=8087 RUN_YZ=1 RUN_DATATYPES=1 RUN_INDEXES=1 ./tox_runner.sh .. @RIAK_TEST_PROTOCOL='http' RIAK_TEST_HTTP_PORT=8098 RUN_YZ=1 RUN_DATATYPES=1 RUN_INDEXES=1 ./tox_runner.sh .. test_security: @echo "Testing Riak Python Client (with security)" - @../setup.py enable_security --riak-admin=$(RIAK_ADMIN) + @$(RIAK_ADMIN) security enable @RIAK_TEST_PROTOCOL='pbc' RIAK_TEST_PB_PORT=8087 RUN_YZ=1 RUN_INDEXES=1 RUN_SECURITY=1 RUN_POOL=0 RUN_RESOLVE=0 ./tox_runner.sh .. @RIAK_TEST_PROTOCOL='http' RIAK_TEST_HTTP_PORT=18098 RUN_YZ=1 RUN_INDEXES=1 RUN_SECURITY=1 RUN_POOL=0 RUN_RESOLVE=0 ./tox_runner.sh .. test_timeseries: @echo "Testing Riak Python Client (timeseries)" - @../setup.py disable_security --riak-admin=${RIAK_ADMIN} + @$(RIAK_ADMIN) security disable @RIAK_TEST_PROTOCOL='pbc' RUN_YZ=0 RUN_DATATYPES=0 RUN_INDEXES=1 RUN_TIMESERIES=1 ./tox_runner.sh .. setup: diff --git a/commands.py b/commands.py index 20ce95bc..ac55b4a8 100644 --- a/commands.py +++ b/commands.py @@ -2,20 +2,15 @@ import os import os.path import re -import shutil from distutils.core import Command from distutils.errors import DistutilsOptionError from distutils.file_util import write_file from distutils import log -from string import Template from subprocess import Popen, PIPE -__all__ = ['build_messages', - 'setup_security', 'enable_security', 'disable_security', - 'setup_timeseries', - 'preconfigure', 'configure'] +__all__ = ['build_messages', 'setup_timeseries'] # Exception classes used by this module. @@ -176,303 +171,6 @@ class setup_timeseries(bucket_type_commands, Command): } -class security_commands(object): - def check_security_command(self, *args): - cmd = self._security_command(*args) - return self.check_output(cmd) - - def run_security_command(self, *args): - self.spawn(self._security_command(*args)) - - def _security_command(self, *args): - cmd = [self.riak_admin, "security"] - if isinstance(args, tuple): - for elem in args: - cmd.extend(elem) - else: - cmd.extend(args) - return cmd - - def check_output(self, *args, **kwargs): - if self.dry_run: - log.info(' '.join(args)) - return bytearray() - else: - return check_output(*args, **kwargs) - - -class setup_security(Command, security_commands): - """ - Sets up security for testing. By default this will create: - - * User `testuser` with password `testpassword` - * User `certuser` with password `certpass` - * Two security sources - * Permissions on - * riak_kv.get - * riak_kv.put - * riak_kv.delete - * riak_kv.index - * riak_kv.list_keys - * riak_kv.list_buckets - * riak_kv.mapreduce - * riak_core.get_bucket - * riak_core.set_bucket - * riak_core.get_bucket_type - * riak_core.set_bucket_type - * search.admin - * search.query - """ - - description = "create security settings used in integration tests" - - user_options = [ - ('riak-admin=', None, 'path to the riak-admin script'), - ('username=', None, 'test user account'), - ('password=', None, 'password for test user account'), - ('certuser=', None, 'certificate test user account'), - ('certpass=', None, 'password for certificate test user account') - ] - - _commands = [ - "add-user $USERNAME password=$PASSWORD", - "add-source $USERNAME 127.0.0.1/32 password", - "add-user $CERTUSER password=$CERTPASS", - "add-source $CERTUSER 127.0.0.1/32 certificate" - ] - - _grants = { - "riak_kv.get": ["any"], - "riak_kv.get_preflist": ["any"], - "riak_kv.put": ["any"], - "riak_kv.delete": ["any"], - "riak_kv.index": ["any"], - "riak_kv.list_keys": ["any"], - "riak_kv.list_buckets": ["any"], - "riak_kv.mapreduce": ["any"], - "riak_core.get_bucket": ["any"], - "riak_core.set_bucket": ["any"], - "riak_core.get_bucket_type": ["any"], - "riak_core.set_bucket_type": ["any"], - "search.admin": ["index", "schema"], - "search.query": ["index", "schema"] - } - - def initialize_options(self): - self.riak_admin = None - self.username = None - self.password = None - self.certuser = None - self.certpass = None - - def finalize_options(self): - if self.riak_admin is None: - raise DistutilsOptionError("riak-admin option not set") - if self.username is None: - self.username = 'testuser' - if self.password is None: - self.password = 'testpassword' - if self.certuser is None: - self.certuser = 'certuser' - if self.certpass is None: - self.certpass = 'certpass' - - def run(self): - if self._check_available(): - for cmd in self._commands: - # Replace the username and password if specified - s = Template(cmd) - newcmd = s.substitute(USERNAME=self.username, - PASSWORD=self.password, - CERTUSER=self.certuser, - CERTPASS=self.certpass) - log.info("Security command: {0}".format(repr(newcmd))) - self.run_security_command(tuple(newcmd.split(' '))) - for perm in self._grants: - self._apply_grant(perm, self._grants[perm]) - - def _check_available(self): - try: - self.check_security_command("status") - return True - except CalledProcessError: - log.error("Security is not supported on this Riak node!") - return False - - def _apply_grant(self, perm, targets): - for target in targets: - cmd = ["grant", perm, "on", target, "to", self.username] - log.info("Granting permission {0} on {1} to {2}" - .format(repr(perm), repr(target), repr(self.username))) - self.run_security_command(cmd) - cmd = ["grant", perm, "on", target, "to", self.certuser] - log.info("Granting permission {0} on {1} to {2}" - .format(repr(perm), repr(target), repr(self.certuser))) - self.run_security_command(cmd) - - -class enable_security(Command, security_commands): - """ - Actually turn on security. - """ - description = "turn on security within Riak" - - user_options = [ - ('riak-admin=', None, 'path to the riak-admin script'), - ] - - def initialize_options(self): - self.riak_admin = None - - def finalize_options(self): - if self.riak_admin is None: - raise DistutilsOptionError("riak-admin option not set") - - def run(self): - cmd = "enable" - self.run_security_command(tuple(cmd.split(' '))) - - -class disable_security(Command, security_commands): - """ - Actually turn off security. - """ - description = "turn off security within Riak" - - user_options = [ - ('riak-admin=', None, 'path to the riak-admin script'), - ] - - def initialize_options(self): - self.riak_admin = None - - def finalize_options(self): - if self.riak_admin is None: - raise DistutilsOptionError("riak-admin option not set") - - def run(self): - cmd = "disable" - self.run_security_command(tuple(cmd.split(' '))) - - -class preconfigure(Command): - """ - Sets up security configuration. - - * Update these lines in riak.conf - * storage_backend = leveldb - * search = on - * listener.protobuf.internal = 127.0.0.1:8087 - * listener.http.internal = 127.0.0.1:8098 - * listener.https.internal = 127.0.0.1:18098 - * ssl.certfile = $pwd/tests/resources/server.crt - * ssl.keyfile = $pwd/tests/resources/server.key - * ssl.cacertfile = $pwd/tests/resources/ca.crt - * check_crl = off - """ - - description = "preconfigure security settings used in integration tests" - - user_options = [ - ('riak-conf=', None, 'path to the riak.conf file'), - ('host=', None, 'IP of host running Riak'), - ('pb-port=', None, 'protocol buffers port number'), - ('https-port=', None, 'https port number') - ] - - def initialize_options(self): - self.riak_conf = None - self.host = "127.0.0.1" - self.pb_port = "8087" - self.http_port = "8098" - self.https_port = "18098" - - def finalize_options(self): - if self.riak_conf is None: - raise DistutilsOptionError("riak-conf option not set") - - def run(self): - self.cert_dir = os.path.dirname(os.path.realpath(__file__)) + \ - "/riak/tests/resources" - self._update_riak_conf() - - def _update_riak_conf(self): - http_host = self.host + ':' + self.http_port - https_host = self.host + ':' + self.https_port - pb_host = self.host + ':' + self.pb_port - self._backup_file(self.riak_conf) - conf = None - with open(self.riak_conf, 'r', buffering=1) as f: - conf = f.read() - conf = re.sub(r'search\s+=\s+off', r'search = on', conf) - conf = re.sub(r'##[ ]+ssl\.', r'ssl.', conf) - conf = re.sub(r'ssl.certfile\s+=\s+\S+', - r'ssl.certfile = ' + self.cert_dir + '/server.crt', - conf) - conf = re.sub(r'storage_backend\s+=\s+\S+', - r'storage_backend = leveldb', - conf) - conf = re.sub(r'ssl.keyfile\s+=\s+\S+', - r'ssl.keyfile = ' + self.cert_dir + '/server.key', - conf) - conf = re.sub(r'ssl.cacertfile\s+=\s+\S+', - r'ssl.cacertfile = ' + self.cert_dir + - '/ca.crt', - conf) - conf = re.sub(r'#*[ ]*listener.http.internal\s+=\s+\S+', - r'listener.http.internal = ' + http_host, - conf) - conf = re.sub(r'#*[ ]*listener.https.internal\s+=\s+\S+', - r'listener.https.internal = ' + https_host, - conf) - conf = re.sub(r'listener.protobuf.internal\s+=\s+\S+', - r'listener.protobuf.internal = ' + pb_host, - conf) - conf += 'check_crl = off\n' - # Older versions of OpenSSL client library need to match on the server - conf += 'tls_protocols.tlsv1 = on\n' - conf += 'tls_protocols.tlsv1.1 = on\n' - with open(self.riak_conf, 'w', buffering=1) as f: - f.write(conf) - - def _backup_file(self, name): - backup = name + ".bak" - if os.path.isfile(name): - shutil.copyfile(name, backup) - else: - log.info("Cannot backup missing file {0}".format(repr(name))) - - -class configure(Command): - """ - Sets up security configuration. - - * Run setup_security - """ - - description = "security settings for testing" - - user_options = setup_security.user_options - - def initialize_options(self): - self.riak_admin = None - self.username = None - self.password = None - - def finalize_options(self): - security = self.distribution.get_command_obj('setup_security') - security.riak_admin = self.riak_admin - security.username = self.username - security.password = self.password - - def run(self): - # Run all relevant sub-commands. - for cmd_name in self.get_sub_commands(): - self.run_command(cmd_name) - - sub_commands = [('setup_security', None)] - - class ComparableMixin(object): def _compare(self, other, method): try: diff --git a/riak/tests/__init__.py b/riak/tests/__init__.py index 7ee88648..0cf7d1d4 100644 --- a/riak/tests/__init__.py +++ b/riak/tests/__init__.py @@ -1,4 +1,6 @@ import os +import socket + from riak.test_server import TestServer from riak.security import SecurityCreds @@ -17,6 +19,14 @@ except ImportError: HAVE_PROTO = False + +def hostname_resolves(hostname): + try: + socket.gethostbyname(hostname) + return 1 + except socket.error: + return 0 + HOST = os.environ.get('RIAK_TEST_HOST', '127.0.0.1') PROTOCOL = os.environ.get('RIAK_TEST_PROTOCOL', 'pbc') @@ -45,24 +55,33 @@ RUN_DATATYPES = int(os.environ.get('RUN_DATATYPES', '1')) RUN_SECURITY = int(os.environ.get('RUN_SECURITY', '0')) -SECURITY_USER = os.environ.get('RIAK_TEST_SECURITY_USER', 'testuser') -SECURITY_PASSWD = os.environ.get('RIAK_TEST_SECURITY_PASSWD', 'testpassword') +if RUN_SECURITY: + h = 'riak-test' + if hostname_resolves(h): + HOST = PB_HOST = HTTP_HOST = h + else: + raise AssertionError( + 'RUN_SECURITY requires that the host name' + + ' "riak-test" resolves to the IP address of a Riak node' + + ' with security enabled.') + +SECURITY_USER = os.environ.get('RIAK_TEST_SECURITY_USER', 'riakpass') +SECURITY_PASSWD = os.environ.get('RIAK_TEST_SECURITY_PASSWD', 'Test1234') + SECURITY_CACERT = os.environ.get('RIAK_TEST_SECURITY_CACERT', - 'riak/tests/resources/ca.crt') + 'tools/test-ca/certs/cacert.pem') SECURITY_REVOKED = os.environ.get('RIAK_TEST_SECURITY_REVOKED', - 'riak/tests/resources/server.crl') + 'tools/test-ca/crl/crl.pem') SECURITY_BAD_CERT = os.environ.get('RIAK_TEST_SECURITY_BAD_CERT', - 'riak/tests/resources/bad_ca.crt') + 'tools/test-ca/certs/badcert.pem') # Certificate-based Authentication only supported by PBC -# N.B., username and password must both still be supplied -SECURITY_KEY = os.environ.get('RIAK_TEST_SECURITY_KEY', - 'riak/tests/resources/client.key') +SECURITY_KEY = os.environ.get( + 'RIAK_TEST_SECURITY_KEY', + 'tools/test-ca/private/riakuser-client-cert-key.pem') SECURITY_CERT = os.environ.get('RIAK_TEST_SECURITY_CERT', - 'riak/tests/resources/client.crt') + 'tools/test-ca/certs/riakuser-client-cert.pem') SECURITY_CERT_USER = os.environ.get('RIAK_TEST_SECURITY_CERT_USER', - 'certuser') -SECURITY_CERT_PASSWD = os.environ.get('RIAK_TEST_SECURITY_CERT_PASSWD', - 'certpass') + 'riakuser') SECURITY_CIPHERS = 'DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:' + \ 'DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:AES128-SHA256:' + \ diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index ee35fefa..bfa2b888 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -701,7 +701,7 @@ def test_store_binary_object_from_file(self): def test_store_binary_object_from_file_should_use_default_mimetype(self): bucket = self.client.bucket(self.bucket_name) filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), - os.pardir, os.pardir, 'README.rst') + os.pardir, os.pardir, 'README.md') obj = bucket.new_from_file(self.key_name, filepath) obj.store() obj = bucket.get(self.key_name) diff --git a/riak/tests/test_security.py b/riak/tests/test_security.py index 85588ee0..056c1b48 100644 --- a/riak/tests/test_security.py +++ b/riak/tests/test_security.py @@ -4,8 +4,7 @@ from riak.tests import RUN_SECURITY, SECURITY_USER, SECURITY_PASSWD, \ SECURITY_CACERT, SECURITY_KEY, SECURITY_CERT, SECURITY_REVOKED, \ - SECURITY_CERT_USER, SECURITY_CERT_PASSWD, SECURITY_BAD_CERT, \ - SECURITY_CIPHERS + SECURITY_CERT_USER, SECURITY_BAD_CERT, SECURITY_CIPHERS from riak.security import SecurityCreds from riak.tests.base import IntegrationTestBase @@ -91,7 +90,6 @@ def test_security_password_without_cacert(self): @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is 0') def test_security_cert_authentication(self): creds = SecurityCreds(username=SECURITY_CERT_USER, - password=SECURITY_CERT_PASSWD, ciphers=SECURITY_CIPHERS, cert_file=SECURITY_CERT, pkey_file=SECURITY_KEY, diff --git a/riak/transports/http/__init__.py b/riak/transports/http/__init__.py index 0024589d..d7e69c3d 100644 --- a/riak/transports/http/__init__.py +++ b/riak/transports/http/__init__.py @@ -137,7 +137,6 @@ def connect(self): self.credentials._check_revoked_cert(self.sock) else: ssl_ctx = configure_ssl_context(self.credentials) - host = "riak@" + self.host if self.timeout is not None: sock.settimeout(self.timeout) self.sock = ssl.SSLSocket(sock=sock, @@ -146,7 +145,7 @@ def connect(self): cert_reqs=ssl.CERT_REQUIRED, ca_certs=self.credentials.cacert_file, ciphers=self.credentials.ciphers, - server_hostname=host) + server_hostname=self.host) self.sock.context = ssl_ctx diff --git a/riak/transports/pbc/connection.py b/riak/transports/pbc/connection.py index 12045323..60f264c1 100644 --- a/riak/transports/pbc/connection.py +++ b/riak/transports/pbc/connection.py @@ -84,7 +84,10 @@ def _auth(self): """ req = riak.pb.riak_pb2.RpbAuthReq() req.user = str_to_bytes(self._client._credentials.username) - req.password = str_to_bytes(self._client._credentials.password) + password = self._client._credentials.password + if not password: + password = '' + req.password = str_to_bytes(password) msg_code, _ = self._non_connect_request( riak.pb.messages.MSG_CODE_AUTH_REQ, req, @@ -130,7 +133,7 @@ def _ssl_handshake(self): if credentials: try: ssl_ctx = configure_ssl_context(credentials) - host = "riak@" + self._address[0] + host = self._address[0] ssl_socket = ssl.SSLSocket(sock=self._socket, keyfile=credentials.pkey_file, certfile=credentials.cert_file, diff --git a/riak/transports/security.py b/riak/transports/security.py index dfd4cdc1..a2ac46d5 100644 --- a/riak/transports/security.py +++ b/riak/transports/security.py @@ -72,6 +72,7 @@ def configure_ssl_context(credentials): pkeyfile = certfile if certfile: ssl_ctx.load_cert_chain(certfile, pkeyfile) + # TODO https://bugs.python.org/issue8813 if credentials.crl_file is not None: ssl_ctx.load_verify_locations(credentials.crl_file) ssl_ctx.verify_flags = ssl.VERIFY_CRL_CHECK_LEAF diff --git a/setup.py b/setup.py index 1c40976b..0d9ef61f 100755 --- a/setup.py +++ b/setup.py @@ -3,9 +3,7 @@ import platform from setuptools import setup, find_packages from version import get_version -from commands import preconfigure, configure, \ - setup_security, enable_security, disable_security, setup_timeseries, \ - build_messages +from commands import setup_timeseries, build_messages install_requires = ['six >= 1.8.0'] requires = ['six(>=1.8.0)'] @@ -45,12 +43,7 @@ url='https://github.com/basho/riak-python-client', cmdclass={ 'build_messages': build_messages, - 'setup_timeseries': setup_timeseries, - 'setup_security': setup_security, - 'preconfigure': preconfigure, - 'configure': configure, - 'enable_security': enable_security, - 'disable_security': disable_security + 'setup_timeseries': setup_timeseries }, classifiers=['License :: OSI Approved :: Apache Software License', 'Intended Audience :: Developers', diff --git a/tools b/tools index 343c4a48..c7b46bce 160000 --- a/tools +++ b/tools @@ -1 +1 @@ -Subproject commit 343c4a485e26bcee971a41faf9234d65a3fb8b53 +Subproject commit c7b46bce22cd96ef3c4b0c17f7e156f14029264c From 0905534418310197029eb05303e4aab6b0a44dfb Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Sat, 13 Feb 2016 18:38:48 -0800 Subject: [PATCH 5/7] Remove sections from README that are no longer relevant --- README.md | 60 ++----------------------------------------------------- 1 file changed, 2 insertions(+), 58 deletions(-) diff --git a/README.md b/README.md index 52f41f1e..5542ed2d 100644 --- a/README.md +++ b/README.md @@ -64,52 +64,6 @@ make -C buildbot test That will run the test suite twice: once with security enabled and once without. -Testing Options ---------------- - -If you wish to change the default options you can run the setup by hand. First configure the test node by adjusting the `riak.conf` settings, where `RIAK_DIR` is the path to the top your Riak installation - -```sh -python setup.py preconfigure --riak-conf=$RIAK_DIR/etc/riak.conf -``` - -Optionally the hostname and port numbers can be changed, too, via these arguments: - -* `--host=` IP of host running Riak (default is `localhost`) -* `--pb-port=` protocol buffers port number (default is `8087`) -* `--http-port=` http port number (default is `8098`) -* `--https-port=` https port number (default is `8099`) - -You may alternately add these lines to `setup.cfg` - -```ini -[preconfigure] -riak-conf=/Users/sean/dev/riak/rel/riak/etc/riak.conf -host=localhost -pb-port=8087 -http-port=8098 -https-port=8099 -``` - -Next start the test node. Once it is running, a test configuration is installed which includes security test users and bucket types - -```sh -python setup.py configure --riak-admin=$RIAK_DIR/bin/riak-admin -``` - -Optionally these configuration settings can be changed, too: - -* `--username=` test user account (default is `testuser`) -* `--password=` password for test user account (default is `testpassword`) -* `--certuser=` secruity test user account (default is `certuser`) -* `--certpass=` password for security test user account (default is `certpass`) - -Similarly `setup.cfg` may be modified instead. To run the tests against a Riak server (with configured TCP port configuration) on localhost, execute - -```sh -python setup.py test -``` - Connections to Riak in Tests ---------------------------- @@ -175,19 +129,9 @@ To test [Secondary Indexes](http://docs.basho.com/riak/2.0.0/dev/using/2i/), the Testing Security (Riak 2+) -------------------------- -By default [Security](http://docs.basho.com/riak/2.0.0beta1/ops/running/authz/) is not enabled on Riak. Once `security = on` is configured in the `riak.conf` file it can be enabled with `riak-admin`. - -If you have set up the test environment outlined in the Testing section you can go ahead and use this command to enable security +Ensure that the hostname `riak-test` resolves to your Riak host (most likely `localhost`). This is so the SSL host verification can succeed. -```sh -python setup.py enable_security --riak-admin=$RIAK_DIR/bin/riak-admin -``` - -Once you are done testing security you can also - -```sh -python setup.py disable_security --riak-admin=$RIAK_DIR/bin/riak-admin -``` +By default [Security](http://docs.basho.com/riak/2.0.0beta1/ops/running/authz/) is not enabled on Riak. Once `security = on` is configured in the `riak.conf` file it can be enabled with `riak-admin`. To run the tests From d046ab5daec2d02d2e16d1243a1b4ca92dff85a4 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Sat, 13 Feb 2016 18:43:41 -0800 Subject: [PATCH 6/7] Remove last vestiges of that rst file --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index db8a14a0..93567e45 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,6 @@ include docs/* include riak/erl_src/* -include README.rst +include README.md include LICENSE include RELEASE_NOTES.md include version.py From 550a2d14fcb282af50a5b16e06c275e441956b4a Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Thu, 18 Feb 2016 07:07:54 -0800 Subject: [PATCH 7/7] Update to latest riak-client-tools --- tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools b/tools index c7b46bce..5ff5850e 160000 --- a/tools +++ b/tools @@ -1 +1 @@ -Subproject commit c7b46bce22cd96ef3c4b0c17f7e156f14029264c +Subproject commit 5ff5850e1d7164f4f64f45a31d9b257e01a19e58