diff --git a/.gitignore b/.gitignore index f9515221..24c0bded 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ *.pyc .python-version +.tox/ + docs/_build .*.swp diff --git a/README.rst b/README.rst index 99bfe27b..a04e86ad 100644 --- a/README.rst +++ b/README.rst @@ -139,6 +139,8 @@ If your Riak server isn't running on localhost or you have built a Riak devrel from source, use the environment variables ``RIAK_TEST_HOST``, ``RIAK_TEST_HTTP_PORT`` and ``RIAK_TEST_PB_PORT`` to specify where to find the Riak server. +``RIAK_TEST_PROTOCOL`` to specify which protocol to test. Can be +either ``pbc`` or ``http``. Some of the connection tests need port numbers that are NOT in use. If ports 1023 and 1022 are in use on your test system, set the @@ -150,7 +152,7 @@ Testing Search If you don't have `Riak Search `_ enabled, you -can set the ``SKIP_SEARCH`` environment variable to 1 skip those +can set the ``RUN_SEARCH`` environment variable to 0 skip those tests. If you don't have `Search 2.0 `_ @@ -176,7 +178,37 @@ You may alternately add these lines to `setup.cfg` [create_bucket_types] riak-admin=/Users/sean/dev/riak/rel/riak/bin/riak-admin -To skip the bucket-type tests, set the ``SKIP_BTYPES`` environment +To skip the bucket-type tests, set the ``RUN_BTYPES`` environment +variable to ``0``. + +Testing Data Types (Riak 2+) +---------------------------- + +To test data types, you must set up bucket types (see above.) + +To skip the data type tests, set the ``RUN_DATATYPES`` environment +variable to ``0``. + +Testing Timeseries (Riak 2.1+) +------------------------------ + +To test timeseries data, you must run the ``setup_timeseries`` command, +which will create the bucket-types used in testing, or create them +manually yourself. It can be run like so (substituting ``$RIAK`` with +the root of your Riak install) + +.. code-block:: console + + ./setup.py setup_timeseries --riak-admin=$RIAK/bin/riak-admin + +You may alternately add these lines to `setup.cfg` + +.. code-block:: ini + + [setup_timeseries] + riak-admin=/Users/sean/dev/riak/rel/riak/bin/riak-admin + +To enable the timeseries tests, set the ``RUN_TIMESERIES`` environment variable to ``1``. Testing Secondary Indexes @@ -184,7 +216,7 @@ Testing Secondary Indexes To test `Secondary Indexes `_, -the ``SKIP_INDEX`` environment variable must be set to 0 (or 1 to skip them.) +the ``RUN_INDEXES`` environment variable must be set to 1 (or 0 to skip them.) Testing Security (Riak 2+) -------------------------- @@ -212,3 +244,51 @@ To run the tests, then simply .. code-block:: console RUN_SECURITY=1 RIAK_TEST_HTTP_PORT=18098 python setup.py test + +Contributors +-------------------------- + - Andrew Thompson + - Andy Gross + - Armon Dadgar + - Brett Hazen + - Brett Hoerner + - Brian Roach + - Bryan Fink + - Daniel Lindsley + - Daniel Néri + - Daniel Reverri + - David Koblas + - Dmitry Rozhkov + - Eric Florenzano + - Eric Moritz + - Filip de Waard + - Gilles Devaux + - Greg Nelson + - Greg Stein + - Gregory Burd + - Ian Plosker + - Jayson Baird + - Jeffrey Massung + - Jon Meredith + - Josip Lisec + - Justin Sheehy + - Kevin Smith + - `Luke Bakken `_ + - Mark Erdmann + - Mark Phillips + - Mathias Meyer + - Matt Heitzenroder + - Mikhail Sobolev + - Reid Draper + - Russell Brown + - Rusty Klophaus + - Rusty Klophaus + - Scott Lystig Fritchie + - Sean Cribbs + - Shuhao Wu + - Silas Sewell + - Socrates Lee + - Soren Hansen + - Sreejith Kesavan + - Timothée Peignier + - William Kral diff --git a/THANKS b/THANKS deleted file mode 100644 index 4fccfe7f..00000000 --- a/THANKS +++ /dev/null @@ -1,45 +0,0 @@ -The following people have contributed to the Riak Python client: - -Andrew Thompson -Andy Gross -Armon Dadgar -Brett Hazen -Brett Hoerner -Brian Roach -Bryan Fink -Daniel Lindsley -Daniel Néri -Daniel Reverri -David Koblas -Dmitry Rozhkov -Eric Florenzano -Eric Moritz -Filip de Waard -Gilles Devaux -Greg Nelson -Greg Stein -Gregory Burd -Ian Plosker -Jayson Baird -Jeffrey Massung -Jon Meredith -Josip Lisec -Justin Sheehy -Kevin Smith -Mark Erdmann -Mark Phillips -Mathias Meyer -Matt Heitzenroder -Mikhail Sobolev -Reid Draper -Russell Brown -Rusty Klophaus -Scott Lystig Fritchie -Sean Cribbs -Shuhao Wu -Silas Sewell -Socrates Lee -Soren Hansen -Sreejith Kesavan -Timothée Peignier -William Kral diff --git a/buildbot/Makefile b/buildbot/Makefile index 1323a276..4be4564d 100644 --- a/buildbot/Makefile +++ b/buildbot/Makefile @@ -26,12 +26,19 @@ test: setup test_normal test_security test_normal: @echo "Testing Riak Python Client (without security)" @../setup.py disable_security --riak-admin=${RIAK_ADMIN} - @RUN_YZ=1 SKIP_DATATYPES=0 SKIP_INDEXES=0 ./tox_runner.sh .. + @RIAK_TEST_PROTOCOL='pbc' RUN_YZ=1 RUN_DATATYPES=1 RUN_INDEXES=1 ./tox_runner.sh .. + @RIAK_TEST_PROTOCOL='http' RUN_YZ=1 RUN_DATATYPES=1 RUN_INDEXES=1 ./tox_runner.sh .. test_security: @echo "Testing Riak Python Client (with security)" @../setup.py enable_security --riak-admin=${RIAK_ADMIN} - @RUN_YZ=1 SKIP_INDEXES=0 RUN_SECURITY=1 SKIP_POOL=1 SKIP_RESOLVE=1 RIAK_TEST_HTTP_PORT=18098 ./tox_runner.sh .. + @RIAK_TEST_PROTOCOL='pbc' RUN_YZ=1 RUN_INDEXES=1 RUN_SECURITY=1 RUN_POOL=0 RUN_RESOLVE=0 ./tox_runner.sh .. + @RIAK_TEST_PROTOCOL='http' RUN_YZ=1 RUN_INDEXES=1 RUN_SECURITY=1 RUN_POOL=0 RUN_RESOLVE=0 RIAK_TEST_HTTP_PORT=18098 ./tox_runner.sh .. + +test_timeseries: + @echo "Testing Riak Python Client (timeseries)" + @../setup.py disable_security --riak-admin=${RIAK_ADMIN} + @RIAK_TEST_PROTOCOL='pbc' RUN_YZ=0 RUN_DATATYPES=0 RUN_INDEXES=1 RUN_TIMESERIES=1 ./tox_runner.sh .. # These are required to actually build all the Python versions: # * pip install tox diff --git a/buildbot/tox_cleanup.sh b/buildbot/tox_cleanup.sh new file mode 100755 index 00000000..bd5324c7 --- /dev/null +++ b/buildbot/tox_cleanup.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +for pbin in .tox/*/bin +do + echo $pbin + pip="$pbin/pip" + $pip uninstall riak_pb --yes + $pip uninstall riak --yes + $pip uninstall protobuf --yes + $pip uninstall python3-riak-pb --yes + $pip uninstall python3-protobuf --yes + echo ----- +done diff --git a/buildbot/tox_setup.sh b/buildbot/tox_setup.sh index 1dc3f72c..05a1fe49 100755 --- a/buildbot/tox_setup.sh +++ b/buildbot/tox_setup.sh @@ -47,19 +47,19 @@ if [[ -z $(pyenv versions | grep riak_3.3.6) ]]; then VERSION_ALIAS="riak_3.3.6" pyenv install 3.3.6 pyenv virtualenv riak_3.3.6 riak-py33 fi -if [[ -z $(pyenv versions | grep riak_2.7.10) ]]; then - VERSION_ALIAS="riak_2.7.10" pyenv install 2.7.10 - pyenv virtualenv riak_2.7.10 riak-py27 +if [[ -z $(pyenv versions | grep riak_3.5.1) ]]; then + VERSION_ALIAS="riak_3.5.1" pyenv install 3.5.1 + pyenv virtualenv riak_3.5.1 riak-py35 fi -if [[ -z $(pyenv versions | grep riak_2.7.9) ]]; then - VERSION_ALIAS="riak_2.7.9" pyenv install 2.7.9 - pyenv virtualenv riak_2.7.9 riak-py279 +if [[ -z $(pyenv versions | grep riak_2.7.11) ]]; then + VERSION_ALIAS="riak_2.7.11" pyenv install 2.7.11 + pyenv virtualenv riak_2.7.11 riak-py27 fi -if [[ -z $(pyenv versions | grep riak_2.6.9) ]]; then - VERSION_ALIAS="riak_2.6.9" pyenv install 2.6.9 - pyenv virtualenv riak_2.6.9 riak-py26 +if [[ -z $(pyenv versions | grep riak_2.7.8) ]]; then + VERSION_ALIAS="riak_2.7.8" pyenv install 2.7.8 + pyenv virtualenv riak_2.7.8 riak-py278 fi -pyenv global riak-py34 riak-py33 riak-py27 riak-py279 riak-py26 +pyenv global riak-py34 riak-py33 riak-py35 riak-py27 riak-py278 pyenv versions # Now install tox diff --git a/commands.py b/commands.py index 06ee3039..b3d41ea7 100644 --- a/commands.py +++ b/commands.py @@ -11,8 +11,10 @@ import os.path -__all__ = ['create_bucket_types', 'setup_security', 'enable_security', - 'disable_security', 'preconfigure', 'configure'] +__all__ = ['create_bucket_types', + 'setup_security', 'enable_security', 'disable_security', + 'setup_timeseries', + 'preconfigure', 'configure'] # Exception classes used by this module. @@ -73,35 +75,7 @@ def check_output(*popenargs, **kwargs): import json -class create_bucket_types(Command): - """ - Creates bucket-types appropriate for testing. By default this will create: - - * `pytest-maps` with ``{"datatype":"map"}`` - * `pytest-sets` with ``{"datatype":"set"}`` - * `pytest-counters` with ``{"datatype":"counter"}`` - * `pytest-consistent` with ``{"consistent":true}`` - * `pytest-write-once` with ``{"write_once": true}`` - * `pytest-mr` - * `pytest` with ``{"allow_mult":false}`` - """ - - description = "create bucket-types used in integration tests" - - user_options = [ - ('riak-admin=', None, 'path to the riak-admin script') - ] - - _props = { - 'pytest-maps': {'datatype': 'map'}, - 'pytest-sets': {'datatype': 'set'}, - 'pytest-counters': {'datatype': 'counter'}, - 'pytest-consistent': {'consistent': True}, - 'pytest-write-once': {'write_once': True}, - 'pytest-mr': {}, - 'pytest': {'allow_mult': False} - } - +class bucket_type_commands: def initialize_options(self): self.riak_admin = None @@ -171,6 +145,66 @@ def _btype_command(self, *args): return cmd +class create_bucket_types(bucket_type_commands, Command): + """ + Creates bucket-types appropriate for testing. By default this will create: + + * `pytest-maps` with ``{"datatype":"map"}`` + * `pytest-sets` with ``{"datatype":"set"}`` + * `pytest-counters` with ``{"datatype":"counter"}`` + * `pytest-consistent` with ``{"consistent":true}`` + * `pytest-write-once` with ``{"write_once": true}`` + * `pytest-mr` + * `pytest` with ``{"allow_mult":false}`` + """ + + description = "create bucket-types used in integration tests" + + user_options = [ + ('riak-admin=', None, 'path to the riak-admin script') + ] + + _props = { + 'pytest-maps': {'datatype': 'map'}, + 'pytest-sets': {'datatype': 'set'}, + 'pytest-counters': {'datatype': 'counter'}, + 'pytest-consistent': {'consistent': True}, + 'pytest-write-once': {'write_once': True}, + 'pytest-mr': {}, + 'pytest': {'allow_mult': False} + } + + +class setup_timeseries(bucket_type_commands, Command): + """ + Creates bucket-types appropriate for timeseries. + """ + + description = "create bucket-types used in timeseries tests" + + user_options = [ + ('riak-admin=', None, 'path to the riak-admin script') + ] + + _props = { + 'GeoCheckin': { + 'n_val': 3, + 'table_def': ''' + CREATE TABLE GeoCheckin ( + geohash varchar not null, + user varchar not null, + time timestamp not null, + weather varchar not null, + temperature double, + PRIMARY KEY( + (geohash, user, quantum(time, 15, m)), + geohash, user, time + ) + )''' + } + } + + class security_commands(object): def check_security_command(self, *args): cmd = self._security_command(*args) @@ -396,9 +430,9 @@ def _update_riak_conf(self): https_host = self.host + ':' + self.https_port pb_host = self.host + ':' + self.pb_port self._backup_file(self.riak_conf) - f = open(self.riak_conf, 'r', buffering=1) - conf = f.read() - f.close() + conf = None + with open(self.riak_conf, 'r', buffering=1) as f: + conf = f.read() conf = re.sub(r'search\s+=\s+off', r'search = on', conf) conf = re.sub(r'##[ ]+ssl\.', r'ssl.', conf) conf = re.sub(r'ssl.certfile\s+=\s+\S+', @@ -427,9 +461,8 @@ def _update_riak_conf(self): # Older versions of OpenSSL client library need to match on the server conf += 'tls_protocols.tlsv1 = on\n' conf += 'tls_protocols.tlsv1.1 = on\n' - f = open(self.riak_conf, 'w', buffering=1) - f.write(conf) - f.close() + with open(self.riak_conf, 'w', buffering=1) as f: + f.write(conf) def _backup_file(self, name): backup = name + ".bak" @@ -469,6 +502,4 @@ def run(self): for cmd_name in self.get_sub_commands(): self.run_command(cmd_name) - sub_commands = [('create_bucket_types', None), - ('setup_security', None) - ] + sub_commands = [('create_bucket_types', None), ('setup_security', None)] diff --git a/docs/client.rst b/docs/client.rst index edf9d14a..ae2f8e54 100644 --- a/docs/client.rst +++ b/docs/client.rst @@ -123,6 +123,15 @@ Key-level Operations .. automethod:: RiakClient.fetch_datatype .. automethod:: RiakClient.update_datatype +-------------------- +Timeseries Operations +-------------------- + +.. automethod:: RiakClient.ts_get +.. automethod:: RiakClient.ts_put +.. automethod:: RiakClient.ts_delete +.. automethod:: RiakClient.ts_query + ---------------- Query Operations ---------------- diff --git a/riak/__init__.py b/riak/__init__.py index eddc69bc..9e761a91 100644 --- a/riak/__init__.py +++ b/riak/__init__.py @@ -1,46 +1,23 @@ """ -Copyright 2010 Rusty Klophaus -Copyright 2010 Justin Sheehy -Copyright 2009 Jay Baird - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. ---- The Riak API for Python allows you to connect to a Riak instance, create, modify, and delete Riak objects, add and remove links from Riak objects, run Javascript (and Erlang) based Map/Reduce operations, and run Linkwalking operations. - -See the unit_tests.py file for example usage. - -@author Rusty Klophaus (@rklophaus) (rusty@basho.com) -@author Andy Gross (@argv0) (andy@basho.com) -@author Jon Meredith (@jmeredith) (jmeredith@basho.com) -@author Jay Baird (@skatterbean) (jay@mochimedia.com) """ from riak.riak_error import RiakError, ConflictError from riak.client import RiakClient from riak.bucket import RiakBucket, BucketType +from riak.table import Table from riak.node import RiakNode from riak.riak_object import RiakObject from riak.mapreduce import RiakKeyFilter, RiakMapReduce, RiakLink -__all__ = ['RiakBucket', 'BucketType', 'RiakNode', 'RiakObject', 'RiakClient', - 'RiakMapReduce', 'RiakKeyFilter', 'RiakLink', 'RiakError', - 'ConflictError', 'ONE', 'ALL', 'QUORUM', 'key_filter'] +__all__ = ['RiakBucket', 'Table', 'BucketType', 'RiakNode', + 'RiakObject', 'RiakClient', 'RiakMapReduce', 'RiakKeyFilter', + 'RiakLink', 'RiakError', 'ConflictError', + 'ONE', 'ALL', 'QUORUM', 'key_filter'] ONE = "one" ALL = "all" diff --git a/riak/bucket.py b/riak/bucket.py index 4342d7ad..f6dd3863 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -196,7 +196,7 @@ def new(self, key=None, data=None, content_type='application/json', def get(self, key, r=None, pr=None, timeout=None, include_context=None, basic_quorum=None, notfound_ok=None): """ - Retrieve an :class:`~riak.riak_object.RiakObject` or + Retrieve a :class:`~riak.riak_object.RiakObject` or :class:`~riak.datatypes.Datatype`, based on the presence and value of the :attr:`datatype ` bucket property. @@ -410,7 +410,9 @@ def new_from_file(self, key, filename): :type filename: string :rtype: :class:`RiakObject ` """ - binary_data = open(filename, "rb").read() + binary_data = None + with open(filename, 'rb') as f: + binary_data = f.read() mimetype, encoding = mimetypes.guess_type(filename) if encoding: binary_data = bytearray(binary_data, encoding) diff --git a/riak/client/__init__.py b/riak/client/__init__.py index 002991d8..d623f970 100644 --- a/riak/client/__init__.py +++ b/riak/client/__init__.py @@ -1,24 +1,3 @@ -""" -Copyright 2011 Basho Technologies, Inc. -Copyright 2010 Rusty Klophaus -Copyright 2010 Justin Sheehy -Copyright 2009 Jay Baird - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - try: import simplejson as json except ImportError: @@ -31,6 +10,7 @@ from riak.bucket import RiakBucket, BucketType from riak.mapreduce import RiakMapReduceChain from riak.resolver import default_resolver +from riak.table import Table from riak.transports.http import RiakHttpPool from riak.transports.pbc import RiakPbcPool from riak.security import SecurityCreds @@ -140,6 +120,7 @@ def __init__(self, protocol='pbc', transport_options={}, nodes=None, 'binary/octet-stream': binary_encoder_decoder} self._buckets = WeakValueDictionary() self._bucket_types = WeakValueDictionary() + self._tables = WeakValueDictionary() def _get_protocol(self): return self._protocol @@ -277,12 +258,12 @@ def bucket_type(self, name): not always exist (unlike buckets), but this will always return a :class:`BucketType ` object. - :param name: the bucket name + :param name: the bucket-type name :type name: str :rtype: :class:`BucketType ` """ if not isinstance(name, string_types): - raise TypeError('Bucket name must be a string') + raise TypeError('BucketType name must be a string') if name in self._bucket_types: return self._bucket_types[name] @@ -291,6 +272,26 @@ def bucket_type(self, name): self._bucket_types[name] = btype return btype + def table(self, name): + """ + Gets the table by the specified name. Tables do + not always exist (unlike buckets), but this will always return + a :class:`Table ` object. + + :param name: the table name + :type name: str + :rtype: :class:`Table ` + """ + if not isinstance(name, string_types): + raise TypeError('Table name must be a string') + + if name in self._tables: + return self._tables[name] + else: + table = Table(self, name) + self._tables[name] = table + return table + def close(self): """ Iterate through all of the connections and close each one. diff --git a/riak/client/multiget.py b/riak/client/multiget.py index a8573cc8..20d02801 100644 --- a/riak/client/multiget.py +++ b/riak/client/multiget.py @@ -209,7 +209,9 @@ def multiget(client, keys, **options): client = RiakClient(protocol='pbc') bkeys = [('default', 'multiget', str(key)) for key in range(10000)] - data = open(__file__).read() + data = None + with open(__file__) as f: + data = f.read() print("Benchmarking multiget:") print(" CPUs: {0}".format(cpu_count())) diff --git a/riak/client/operations.py b/riak/client/operations.py index b5972f69..aaecae7d 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -1,26 +1,9 @@ -""" -Copyright 2012 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - from riak.client.transport import RiakClientTransport, \ retryable, retryableHttpOnly from riak.client.multiget import multiget from riak.client.index_page import IndexPage from riak.datatypes import TYPES +from riak.table import Table from riak.util import bytes_to_str from six import string_types, PY2 @@ -553,6 +536,127 @@ def put(self, transport, robj, w=None, dw=None, pw=None, return_body=None, if_none_match=if_none_match, timeout=timeout) + @retryable + def ts_get(self, transport, table, key): + """ + ts_get(table, key) + + Retrieve timeseries value by key + + .. note:: This request is automatically retried :attr:`retries` + times if it fails due to network error. + + :param table: The timeseries table. + :type table: string or :class:`Table ` + :param key: The timeseries value's key. + :type key: list + :rtype: :class:`TsObject ` + """ + t = table + if isinstance(t, string_types): + t = Table(self, table) + return transport.ts_get(t, key) + + @retryable + def ts_put(self, transport, tsobj): + """ + ts_put(tsobj) + + Stores time series data in the Riak cluster. + + .. note:: This request is automatically retried :attr:`retries` + times if it fails due to network error. + + :param tsobj: the time series object to store + :type tsobj: RiakTsObject + :rtype: boolean + """ + return transport.ts_put(tsobj) + + @retryable + def ts_delete(self, transport, table, key): + """ + ts_delete(table, key) + + Delete timeseries value by key + + .. note:: This request is automatically retried :attr:`retries` + times if it fails due to network error. + + :param table: The timeseries table. + :type table: string or :class:`Table ` + :param key: The timeseries value's key. + :type key: list or dict + :rtype: boolean + """ + t = table + if isinstance(t, string_types): + t = Table(self, table) + return transport.ts_delete(t, key) + + @retryable + def ts_query(self, transport, table, query, interpolations=None): + """ + ts_query(table, query, interpolations=None) + + Queries time series data in the Riak cluster. + + .. note:: This request is automatically retried :attr:`retries` + times if it fails due to network error. + + :param table: The timeseries table. + :type table: string or :class:`Table ` + :param query: The timeseries query. + :type query: string + :rtype: :class:`TsObject ` + """ + t = table + if isinstance(t, string_types): + t = Table(self, table) + return transport.ts_query(t, query, interpolations) + + def ts_stream_keys(self, table, timeout=None): + """ + Lists all keys in a time series table via a stream. This is a + generator method which should be iterated over. + + The caller should explicitly close the returned iterator, + either using :func:`contextlib.closing` or calling ``close()`` + explicitly. Consuming the entire iterator will also close the + stream. If it does not, the associated connection might + not be returned to the pool. Example:: + + from contextlib import closing + + # Using contextlib.closing + with closing(client.ts_stream_keys(mytable)) as keys: + for key_list in keys: + do_something(key_list) + + # Explicit close() + stream = client.ts_stream_keys(mytable) + for key_list in stream: + do_something(key_list) + stream.close() + + :param table: the table from which to stream keys + :type table: Table + :param timeout: a timeout value in milliseconds + :type timeout: int + :rtype: iterator + """ + _validate_timeout(timeout) + resource = self._acquire() + transport = resource.object + stream = transport.ts_stream_keys(table, timeout) + stream.attach(resource) + try: + for keylist in stream: + if len(keylist) > 0: + yield keylist + finally: + stream.close() + @retryable def get(self, transport, robj, r=None, pr=None, timeout=None, basic_quorum=None, notfound_ok=None): diff --git a/riak/riak_object.py b/riak/riak_object.py index 2db8b5e8..ab7dd375 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -1,23 +1,3 @@ -""" -Copyright 2012-2013 Basho Technologies -Copyright 2010 Rusty Klophaus -Copyright 2010 Justin Sheehy -Copyright 2009 Jay Baird - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" from riak import ConflictError from riak.content import RiakContent import base64 diff --git a/riak/security.py b/riak/security.py index 542ff225..c1b36123 100644 --- a/riak/security.py +++ b/riak/security.py @@ -37,11 +37,11 @@ # For Python 2.7 and Python 3.x sslver = ssl.OPENSSL_VERSION_NUMBER # Be sure to use at least OpenSSL 1.0.1g - if sslver < OPENSSL_VERSION_101G or \ - not hasattr(ssl, 'PROTOCOL_TLSv1_2'): + tls_12 = hasattr(ssl, 'PROTOCOL_TLSv1_2') + if sslver < OPENSSL_VERSION_101G or not tls_12: verstring = ssl.OPENSSL_VERSION - msg = "Found {0} version, but expected at least OpenSSL 1.0.1g. " \ - "Security may not support TLS 1.2.".format(verstring) + msg = "{0} (>= 1.0.1g required), TLS 1.2 support: {1}" \ + .format(verstring, tls_12) warnings.warn(msg, UserWarning) if hasattr(ssl, 'PROTOCOL_TLSv1_2'): DEFAULT_TLS_VERSION = ssl.PROTOCOL_TLSv1_2 @@ -56,11 +56,11 @@ # For Python 2.6 sslver = OpenSSL.SSL.OPENSSL_VERSION_NUMBER # Be sure to use at least OpenSSL 1.0.1g - if (sslver < OPENSSL_VERSION_101G) or \ - not hasattr(OpenSSL.SSL, 'TLSv1_2_METHOD'): + tls_12 = hasattr(OpenSSL.SSL, 'TLSv1_2_METHOD') + if (sslver < OPENSSL_VERSION_101G) or tls_12: verstring = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION) - msg = "Found {0} version, but expected at least OpenSSL 1.0.1g. " \ - "Security may not support TLS 1.2.".format(verstring) + msg = "{0} (>= 1.0.1g required), TLS 1.2 support: {1}" \ + .format(verstring, tls_12) warnings.warn(msg, UserWarning) if hasattr(OpenSSL.SSL, 'TLSv1_2_METHOD'): DEFAULT_TLS_VERSION = OpenSSL.SSL.TLSv1_2_METHOD diff --git a/riak/table.py b/riak/table.py new file mode 100644 index 00000000..c477a32b --- /dev/null +++ b/riak/table.py @@ -0,0 +1,88 @@ +from six import string_types, PY2 + + +class Table(object): + """ + The ``Table`` object allows you to access properties on a Riak + timeseries table and query timeseries data. + """ + def __init__(self, client, name): + """ + Returns a new ``Table`` instance. + + :param client: A :class:`RiakClient ` + instance + :type client: :class:`RiakClient ` + :param name: The table's name + :type name: string + """ + if not isinstance(name, string_types): + raise TypeError('Table name must be a string') + + if PY2: + try: + name = name.encode('ascii') + except UnicodeError: + raise TypeError('Unicode table names are not supported.') + + self._client = client + self.name = name + + def __str__(self): + return self.name + + def __repr__(self): + return self.name + + def new(self, rows, columns=None): + """ + A shortcut for manually instantiating a new + :class:`~riak.ts_object.TsObject` + + :param rows: An list of lists with timeseries data + :type rows: list + :param columns: An list of Column names and types. Optional. + :type columns: list + :rtype: :class:`~riak.ts_object.TsObject` + """ + from riak.ts_object import TsObject + + return TsObject(self._client, self, rows, columns) + + def get(self, key): + """ + Gets a value from a timeseries table. + + :param key: The timeseries value's key. + :type key: list + :rtype: :class:`TsObject ` + """ + return self._client.ts_get(self, key) + + def delete(self, key): + """ + Deletes a value from a timeseries table. + + :param key: The timeseries value's key. + :type key: list or dict + :rtype: boolean + """ + return self._client.ts_delete(self, key) + + def query(self, query, interpolations=None): + """ + Queries a timeseries table. + + :param query: The timeseries query. + :type query: string + :rtype: :class:`TsObject ` + """ + return self._client.ts_query(self, query, interpolations) + + def stream_keys(self, timeout=None): + """ + Streams keys from a timeseries table. + + :rtype: list + """ + return self._client.ts_stream_keys(self, timeout) diff --git a/riak/tests/__init__.py b/riak/tests/__init__.py index d85447ff..f5aa6866 100644 --- a/riak/tests/__init__.py +++ b/riak/tests/__init__.py @@ -19,6 +19,8 @@ HOST = os.environ.get('RIAK_TEST_HOST', '127.0.0.1') +PROTOCOL = os.environ.get('RIAK_TEST_PROTOCOL', 'pbc') + PB_HOST = os.environ.get('RIAK_TEST_PB_HOST', HOST) PB_PORT = int(os.environ.get('RIAK_TEST_PB_PORT', '8087')) @@ -30,15 +32,17 @@ DUMMY_HTTP_PORT = int(os.environ.get('DUMMY_HTTP_PORT', '1023')) DUMMY_PB_PORT = int(os.environ.get('DUMMY_PB_PORT', '1022')) - -SKIP_SEARCH = int(os.environ.get('SKIP_SEARCH', '1')) +RUN_SEARCH = int(os.environ.get('RUN_SEARCH', '0')) RUN_YZ = int(os.environ.get('RUN_YZ', '0')) -SKIP_INDEXES = int(os.environ.get('SKIP_INDEXES', '1')) +RUN_INDEXES = int(os.environ.get('RUN_INDEXES', '0')) + +RUN_TIMESERIES = int(os.environ.get('RUN_TIMESERIES', '0')) -SKIP_POOL = os.environ.get('SKIP_POOL') -SKIP_RESOLVE = int(os.environ.get('SKIP_RESOLVE', '0')) -SKIP_BTYPES = int(os.environ.get('SKIP_BTYPES', '0')) +RUN_POOL = int(os.environ.get('RUN_POOL', '0')) +RUN_RESOLVE = int(os.environ.get('RUN_RESOLVE', '1')) +RUN_BTYPES = int(os.environ.get('RUN_BTYPES', '1')) +RUN_DATATYPES = int(os.environ.get('RUN_DATATYPES', '1')) RUN_SECURITY = int(os.environ.get('RUN_SECURITY', '0')) SECURITY_USER = os.environ.get('RIAK_TEST_SECURITY_USER', 'testuser') @@ -60,7 +64,9 @@ SECURITY_CERT_PASSWD = os.environ.get('RIAK_TEST_SECURITY_CERT_PASSWD', 'certpass') -SECURITY_CIPHERS = 'DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:AES128-SHA256:AES128-SHA:AES256-SHA256:AES256-SHA:RC4-SHA' +SECURITY_CIPHERS = 'DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:' + \ + 'DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:' + \ + 'AES128-SHA256:AES128-SHA:AES256-SHA256:AES256-SHA:RC4-SHA' SECURITY_CREDS = None if RUN_SECURITY: @@ -68,4 +74,3 @@ password=SECURITY_PASSWD, cacert_file=SECURITY_CACERT, ciphers=SECURITY_CIPHERS) -SKIP_DATATYPES = int(os.environ.get('SKIP_DATATYPES', '0')) diff --git a/riak/tests/base.py b/riak/tests/base.py new file mode 100644 index 00000000..ec0f397c --- /dev/null +++ b/riak/tests/base.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +import logging +import os +import random +import sys + +from riak.client import RiakClient +from riak.tests import HOST, PROTOCOL, PB_PORT, HTTP_PORT, SECURITY_CREDS + + +class IntegrationTestBase(object): + + host = None + pb_port = None + http_port = None + credentials = None + + @staticmethod + def randint(): + return random.randint(1, 999999) + + @staticmethod + def randname(length=12): + out = '' + for i in range(length): + out += chr(random.randint(ord('a'), ord('z'))) + return out + + @classmethod + def create_client(cls, host=None, http_port=None, pb_port=None, + protocol=None, credentials=None, **client_args): + host = host or HOST + http_port = http_port or HTTP_PORT + pb_port = pb_port or PB_PORT + + if protocol is None: + if hasattr(cls, 'protocol') and (cls.protocol is not None): + protocol = cls.protocol + else: + protocol = PROTOCOL + + cls.protocol = protocol + + credentials = credentials or SECURITY_CREDS + + if hasattr(cls, 'logging_enabled') and cls.logging_enabled: + cls.logger.debug("RiakClient(protocol='%s', host='%s', " + + "pb_port='%d', http_port='%d', " + + "credentials='%s', client_args='%s')", + protocol, + host, + pb_port, + http_port, + credentials, + client_args) + + return RiakClient(protocol=protocol, + host=host, + http_port=http_port, + credentials=credentials, + pb_port=pb_port, **client_args) + + @classmethod + def setUpClass(cls): + cls.logging_enabled = False + distutils_debug = os.environ.get('DISTUTILS_DEBUG', '0') + if distutils_debug == '1': + cls.logging_enabled = True + cls.logger = logging.getLogger() + cls.logger.level = logging.DEBUG + cls.logging_stream_handler = logging.StreamHandler(sys.stdout) + cls.logger.addHandler(cls.logging_stream_handler) + + @classmethod + def tearDownClass(cls): + if hasattr(cls, 'logging_enabled') and cls.logging_enabled: + cls.logger.removeHandler(cls.logging_stream_handler) + cls.logging_enabled = False + + def setUp(self): + self.bucket_name = self.randname() + self.key_name = self.randname() + self.client = self.create_client() + + def tearDown(self): + self.client.close() diff --git a/riak/tests/test_six.py b/riak/tests/comparison.py similarity index 88% rename from riak/tests/test_six.py rename to riak/tests/comparison.py index c68dd150..30cde091 100644 --- a/riak/tests/test_six.py +++ b/riak/tests/comparison.py @@ -1,20 +1,4 @@ -""" -Copyright 2014 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" +# -*- coding: utf-8 -*- from six import PY2, PY3 import collections import warnings diff --git a/riak/tests/test_2i.py b/riak/tests/test_2i.py index 86d14999..d7b254e3 100644 --- a/riak/tests/test_2i.py +++ b/riak/tests/test_2i.py @@ -1,32 +1,16 @@ # -*- coding: utf-8 -*- -""" -Copyright 2015 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - import platform from riak import RiakError -from . import SKIP_INDEXES +from riak.tests import RUN_INDEXES +from riak.tests.base import IntegrationTestBase + if platform.python_version() < '2.7': unittest = __import__('unittest2') else: import unittest -class TwoITests(object): +class TwoITests(IntegrationTestBase, unittest.TestCase): def is_2i_supported(self): # Immediate test to see if 2i is even supported w/ the backend try: @@ -37,7 +21,7 @@ def is_2i_supported(self): return False return True # it failed, but is supported! - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_secondary_index_store(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I not supported") @@ -118,7 +102,7 @@ def test_secondary_index_store(self): # Clean up... bucket.get('mykey1').delete() - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_set_indexes(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I not supported") @@ -136,7 +120,7 @@ def test_set_indexes(self): self.assertEqual(1, len(result)) self.assertEqual('foo', str(result[0])) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_remove_indexes(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I not supported") @@ -196,7 +180,7 @@ def test_remove_indexes(self): self.assertEqual(1, len([x for x in bar.indexes if x[0] == 'baz_bin'])) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_secondary_index_query(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I not supported") @@ -225,7 +209,7 @@ def test_secondary_index_query(self): self.assertEqual(3, len(results)) self.assertEqual(set([o2.key, o3.key, o4.key]), vals) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_secondary_index_invalid_name(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I not supported") @@ -235,7 +219,7 @@ def test_secondary_index_invalid_name(self): with self.assertRaises(RiakError): bucket.new('k', 'a').add_index('field1', 'value1') - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_set_index(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I not supported") @@ -253,7 +237,7 @@ def test_set_index(self): obj.set_index('bar2_int', 10) self.assertEqual(set((('bar_int', 3), ('bar2_int', 10))), obj.indexes) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_stream_index(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I not supported") @@ -266,7 +250,7 @@ def test_stream_index(self): self.assertEqual(sorted([o1.key, o2.key, o3.key]), sorted(keys)) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_index_return_terms(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I is not supported") @@ -290,7 +274,7 @@ def test_index_return_terms(self): self.assertEqual([(1002, o2.key), (1003, o3.key), (1004, o4.key)], sorted(spairs)) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_index_pagination(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I is not supported") @@ -325,7 +309,7 @@ def test_index_pagination(self): self.assertEqual(3, pagecount) self.assertEqual([o1.key, o2.key, o3.key, o4.key], presults) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_index_pagination_return_terms(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I is not supported") @@ -350,7 +334,7 @@ def test_index_pagination_return_terms(self): self.assertLessEqual(2, len(results)) self.assertEqual([('val3', o3.key), ('val4', o4.key)], page2) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_index_pagination_stream(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I is not supported") @@ -393,7 +377,7 @@ def test_index_pagination_stream(self): self.assertEqual(3, pagecount) self.assertEqual([o1.key, o2.key, o3.key, o4.key], presults) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_index_pagination_stream_return_terms(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I is not supported") @@ -425,7 +409,7 @@ def test_index_pagination_stream_return_terms(self): self.assertLessEqual(2, len(results)) self.assertEqual([('val3', o3.key), ('val4', o4.key)], results) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_index_eq_query_return_terms(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I is not supported") @@ -435,7 +419,7 @@ def test_index_eq_query_return_terms(self): results = bucket.get_index('field2_int', 1001, return_terms=True) self.assertEqual([(1001, o1.key)], results) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_index_eq_query_stream_return_terms(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I is not supported") @@ -448,7 +432,7 @@ def test_index_eq_query_stream_return_terms(self): self.assertEqual([(1001, o1.key)], results) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_index_timeout(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I is not supported") @@ -467,7 +451,7 @@ def test_index_timeout(self): self.assertEqual([o1.key], bucket.get_index('field1_bin', 'val1', timeout='infinity')) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_index_regex(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I is not supported") @@ -482,7 +466,7 @@ def test_index_regex(self): self.assertEqual([('val2', o2.key)], results) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_index_falsey_endkey_gh378(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I is not supported") diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py deleted file mode 100644 index 2a6ef8cc..00000000 --- a/riak/tests/test_all.py +++ /dev/null @@ -1,442 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Copyright 2015 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" -import random -import platform -from six import PY2 -from threading import Thread - -from riak import RiakError -from riak.client import RiakClient -from riak.riak_object import RiakObject - -from riak.tests.test_yokozuna import YZSearchTests -from riak.tests.test_search import SearchTests, \ - EnableSearchTests, SolrSearchTests -from riak.tests.test_mapreduce import MapReduceAliasTests, \ - ErlangMapReduceTests, JSMapReduceTests, LinkTests, MapReduceStreamTests -from riak.tests.test_kv import BasicKVTests, KVFileTests, \ - BucketPropsTest, CounterTests -from riak.tests.test_2i import TwoITests -from riak.tests.test_btypes import BucketTypeTests -from riak.tests.test_security import SecurityTests -from riak.tests.test_datatypes import DatatypeIntegrationTests - -from riak.tests import HOST, PB_HOST, PB_PORT, HTTP_HOST, HTTP_PORT, \ - HAVE_PROTO, DUMMY_HTTP_PORT, DUMMY_PB_PORT, \ - SKIP_SEARCH, RUN_YZ, SECURITY_CREDS, SKIP_POOL, test_six - -if PY2: - from Queue import Queue -else: - from queue import Queue - -if platform.python_version() < '2.7': - unittest = __import__('unittest2') -else: - import unittest - -testrun_search_bucket = None -testrun_props_bucket = None -testrun_sibs_bucket = None -testrun_yz = {'btype': None, 'bucket': None, 'index': None} -testrun_yz_index = {'btype': None, 'bucket': None, 'index': None} -testrun_yz_mr = {'btype': None, 'bucket': None, 'index': None} - - -def setUpModule(): - global testrun_search_bucket, testrun_props_bucket, \ - testrun_sibs_bucket, testrun_yz, testrun_yz_index, testrun_yz_mr - - c = RiakClient(host=PB_HOST, http_port=HTTP_PORT, - pb_port=PB_PORT, credentials=SECURITY_CREDS) - - testrun_props_bucket = 'propsbucket' - testrun_sibs_bucket = 'sibsbucket' - c.bucket(testrun_sibs_bucket).allow_mult = True - - if (not SKIP_SEARCH and not RUN_YZ): - testrun_search_bucket = 'searchbucket' - b = c.bucket(testrun_search_bucket) - b.enable_search() - - if RUN_YZ: - # YZ index on bucket of the same name - testrun_yz = {'btype': None, 'bucket': 'yzbucket', - 'index': 'yzbucket'} - # YZ index on bucket of a different name - testrun_yz_index = {'btype': None, 'bucket': 'yzindexbucket', - 'index': 'yzindex'} - # Add bucket and type for Search 2.0 -> MapReduce - testrun_yz_mr = {'btype': 'pytest-mr', 'bucket': 'mrbucket', - 'index': 'mrbucket'} - - for yz in (testrun_yz, testrun_yz_index, testrun_yz_mr): - c.create_search_index(yz['index'], timeout=30000) - if yz['btype'] is not None: - t = c.bucket_type(yz['btype']) - b = t.bucket(yz['bucket']) - else: - b = c.bucket(yz['bucket']) - # Keep trying to set search bucket property until it succeeds - index_set = False - while not index_set: - try: - b.set_property('search_index', yz['index']) - index_set = True - except RiakError: - pass - - -def tearDownModule(): - global testrun_search_bucket, testrun_props_bucket, \ - testrun_sibs_bucket, testrun_yz_bucket - - c = RiakClient(host=HTTP_HOST, http_port=HTTP_PORT, - pb_port=PB_PORT, credentials=SECURITY_CREDS) - - c.bucket(testrun_sibs_bucket).clear_properties() - c.bucket(testrun_props_bucket).clear_properties() - - if not SKIP_SEARCH and not RUN_YZ: - b = c.bucket(testrun_search_bucket) - b.clear_properties() - - if RUN_YZ: - for yz in (testrun_yz, testrun_yz_index, testrun_yz_mr): - if yz['btype'] is not None: - t = c.bucket_type(yz['btype']) - b = t.bucket(yz['bucket']) - else: - b = c.bucket(yz['bucket']) - b.set_property('search_index', '_dont_index_') - c.delete_search_index(yz['index']) - for keys in b.stream_keys(): - for key in keys: - b.delete(key) - - -class BaseTestCase(object): - - host = None - pb_port = None - http_port = None - credentials = None - - @staticmethod - def randint(): - return random.randint(1, 999999) - - @staticmethod - def randname(length=12): - out = '' - for i in range(length): - out += chr(random.randint(ord('a'), ord('z'))) - return out - - def create_client(self, host=None, http_port=None, pb_port=None, - protocol=None, credentials=None, - **client_args): - host = host or self.host or HOST - http_port = http_port or self.http_port or HTTP_PORT - pb_port = pb_port or self.pb_port or PB_PORT - protocol = protocol or self.protocol - credentials = credentials or SECURITY_CREDS - return RiakClient(protocol=protocol, - host=host, - http_port=http_port, - credentials=credentials, - pb_port=pb_port, **client_args) - - def setUp(self): - self.bucket_name = self.randname() - self.key_name = self.randname() - self.search_bucket = testrun_search_bucket - self.sibs_bucket = testrun_sibs_bucket - self.props_bucket = testrun_props_bucket - self.yz = testrun_yz - self.yz_index = testrun_yz_index - self.yz_mr = testrun_yz_mr - self.credentials = SECURITY_CREDS - - self.client = self.create_client() - - -class ClientTests(object): - def test_request_retries(self): - # We guess at some ports that will be unused by Riak or - # anything else. - client = self.create_client(http_port=DUMMY_HTTP_PORT, - pb_port=DUMMY_PB_PORT) - - # If retries are exhausted, the final result should also be an - # error. - self.assertRaises(IOError, client.ping) - - def test_request_retries_configurable(self): - # We guess at some ports that will be unused by Riak or - # anything else. - client = self.create_client(http_port=DUMMY_HTTP_PORT, - pb_port=DUMMY_PB_PORT) - - # Change the retry count - client.retries = 10 - self.assertEqual(10, client.retries) - - # The retry count should be a thread local - retries = Queue() - - def _target(): - retries.put(client.retries) - retries.join() - - th = Thread(target=_target) - th.start() - self.assertEqual(3, retries.get(block=True)) - retries.task_done() - th.join() - - # Modify the retries in a with statement - with client.retry_count(5): - self.assertEqual(5, client.retries) - self.assertRaises(IOError, client.ping) - - def test_timeout_validation(self): - bucket = self.client.bucket(self.bucket_name) - key = self.key_name - obj = bucket.new(key) - for bad in [0, -1, False, "foo"]: - with self.assertRaises(ValueError): - self.client.get_buckets(timeout=bad) - - with self.assertRaises(ValueError): - for i in self.client.stream_buckets(timeout=bad): - pass - - with self.assertRaises(ValueError): - self.client.get_keys(bucket, timeout=bad) - - with self.assertRaises(ValueError): - for i in self.client.stream_keys(bucket, timeout=bad): - pass - - with self.assertRaises(ValueError): - self.client.put(obj, timeout=bad) - - with self.assertRaises(ValueError): - self.client.get(obj, timeout=bad) - - with self.assertRaises(ValueError): - self.client.delete(obj, timeout=bad) - - with self.assertRaises(ValueError): - self.client.mapred([], [], bad) - - with self.assertRaises(ValueError): - for i in self.client.stream_mapred([], [], bad): - pass - - with self.assertRaises(ValueError): - self.client.get_index(bucket, 'field1_bin', 'val1', 'val4', - timeout=bad) - - with self.assertRaises(ValueError): - for i in self.client.stream_index(bucket, 'field1_bin', 'val1', - 'val4', timeout=bad): - pass - - def test_multiget_bucket(self): - """ - Multiget operations can be invoked on buckets. - """ - keys = [self.key_name, self.randname(), self.randname()] - for key in keys: - if PY2: - self.client.bucket(self.bucket_name)\ - .new(key, encoded_data=key, content_type="text/plain")\ - .store() - else: - self.client.bucket(self.bucket_name)\ - .new(key, data=key, - content_type="text/plain").store() - results = self.client.bucket(self.bucket_name).multiget(keys) - for obj in results: - self.assertIsInstance(obj, RiakObject) - self.assertTrue(obj.exists) - if PY2: - self.assertEqual(obj.key, obj.encoded_data) - else: - self.assertEqual(obj.key, obj.data) - - def test_multiget_errors(self): - """ - Unrecoverable errors are captured along with the bucket/key - and not propagated. - """ - keys = [self.key_name, self.randname(), self.randname()] - client = self.create_client(http_port=DUMMY_HTTP_PORT, - pb_port=DUMMY_PB_PORT) - results = client.bucket(self.bucket_name).multiget(keys) - for failure in results: - self.assertIsInstance(failure, tuple) - self.assertEqual(failure[0], 'default') - self.assertEqual(failure[1], self.bucket_name) - self.assertIn(failure[2], keys) - if PY2: - self.assertIsInstance(failure[3], StandardError) # noqa - else: - self.assertIsInstance(failure[3], Exception) - - def test_multiget_notfounds(self): - """ - Not founds work in multiget just the same as get. - """ - keys = [("default", self.bucket_name, self.key_name), - ("default", self.bucket_name, self.randname())] - results = self.client.multiget(keys) - for obj in results: - self.assertIsInstance(obj, RiakObject) - self.assertFalse(obj.exists) - - def test_multiget_pool_size(self): - """ - The pool size for multigets can be configured at client initiation - time. Multiget still works as expected. - """ - client = self.create_client(multiget_pool_size=2) - self.assertEqual(2, client._multiget_pool._size) - - keys = [self.key_name, self.randname(), self.randname()] - for key in keys: - if PY2: - client.bucket(self.bucket_name)\ - .new(key, encoded_data=key, content_type="text/plain")\ - .store() - else: - client.bucket(self.bucket_name)\ - .new(key, data=key, content_type="text/plain")\ - .store() - - results = client.bucket(self.bucket_name).multiget(keys) - for obj in results: - self.assertIsInstance(obj, RiakObject) - self.assertTrue(obj.exists) - if PY2: - self.assertEqual(obj.key, obj.encoded_data) - else: - self.assertEqual(obj.key, obj.data) - - @unittest.skipIf(SKIP_POOL, 'SKIP_POOL is set') - def test_pool_close(self): - """ - Iterate over the connection pool and close all connections. - """ - # Do something to add to the connection pool - self.test_multiget_bucket() - if self.client.protocol == 'pbc': - self.assertGreater(len(self.client._pb_pool.resources), 1) - else: - self.assertGreater(len(self.client._http_pool.resources), 1) - # Now close them all up - self.client.close() - self.assertEqual(len(self.client._http_pool.resources), 0) - self.assertEqual(len(self.client._pb_pool.resources), 0) - - -class RiakPbcTransportTestCase(BasicKVTests, - KVFileTests, - BucketPropsTest, - TwoITests, - LinkTests, - ErlangMapReduceTests, - JSMapReduceTests, - MapReduceAliasTests, - MapReduceStreamTests, - EnableSearchTests, - SearchTests, - YZSearchTests, - ClientTests, - CounterTests, - BucketTypeTests, - SecurityTests, - DatatypeIntegrationTests, - BaseTestCase, - unittest.TestCase, - test_six.Comparison): - - def setUp(self): - if not HAVE_PROTO: - self.skipTest('protobuf is unavailable') - self.host = PB_HOST - self.pb_port = PB_PORT - self.protocol = 'pbc' - super(RiakPbcTransportTestCase, self).setUp() - - def test_uses_client_id_if_given(self): - zero_client_id = "\0\0\0\0" - c = self.create_client(client_id=zero_client_id) - self.assertEqual(zero_client_id, c.client_id) - - -class RiakHttpTransportTestCase(BasicKVTests, - KVFileTests, - BucketPropsTest, - TwoITests, - LinkTests, - ErlangMapReduceTests, - JSMapReduceTests, - MapReduceAliasTests, - MapReduceStreamTests, - EnableSearchTests, - SolrSearchTests, - SearchTests, - YZSearchTests, - ClientTests, - CounterTests, - BucketTypeTests, - SecurityTests, - DatatypeIntegrationTests, - BaseTestCase, - unittest.TestCase, - test_six.Comparison): - - def setUp(self): - self.host = HTTP_HOST - self.http_port = HTTP_PORT - self.protocol = 'http' - super(RiakHttpTransportTestCase, self).setUp() - - def test_no_returnbody(self): - bucket = self.client.bucket(self.bucket_name) - o = bucket.new(self.key_name, "bar").store(return_body=False) - self.assertEqual(o.vclock, None) - - def test_too_many_link_headers_shouldnt_break_http(self): - bucket = self.client.bucket(self.bucket_name) - o = bucket.new("lots_of_links", "My god, it's full of links!") - for i in range(0, 300): - link = ("other", "key%d" % i, "next") - o.add_link(link) - - o.store() - stored_object = bucket.get("lots_of_links") - self.assertEqual(len(stored_object.links), 300) - - -if __name__ == '__main__': - unittest.main() diff --git a/riak/tests/test_btypes.py b/riak/tests/test_btypes.py index b3ea5db2..3c8b6c1e 100644 --- a/riak/tests/test_btypes.py +++ b/riak/tests/test_btypes.py @@ -1,25 +1,9 @@ -""" -Copyright 2015 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - import platform -from . import SKIP_BTYPES -from riak.bucket import RiakBucket, BucketType from riak import RiakError, RiakObject +from riak.bucket import RiakBucket, BucketType +from riak.tests import RUN_BTYPES +from riak.tests.base import IntegrationTestBase +from riak.tests.comparison import Comparison if platform.python_version() < '2.7': unittest = __import__('unittest2') @@ -27,7 +11,8 @@ import unittest -class BucketTypeTests(object): +@unittest.skipUnless(RUN_BTYPES, "RUN_BTYPES is 0") +class BucketTypeTests(IntegrationTestBase, unittest.TestCase, Comparison): def test_btype_init(self): btype = self.client.bucket_type('foo') self.assertIsInstance(btype, BucketType) @@ -57,7 +42,6 @@ def test_btype_repr(self): self.assertEqual("", repr(defbtype)) self.assertEqual("", repr(othertype)) - @unittest.skipIf(SKIP_BTYPES == '1', "SKIP_BTYPES is set") def test_btype_get_props(self): defbtype = self.client.bucket_type("default") btype = self.client.bucket_type("pytest") @@ -69,7 +53,6 @@ def test_btype_get_props(self): self.assertIn('n_val', props) self.assertEqual(3, props['n_val']) - @unittest.skipIf(SKIP_BTYPES == '1', "SKIP_BTYPES is set") def test_btype_set_props(self): defbtype = self.client.bucket_type("default") btype = self.client.bucket_type("pytest") @@ -88,13 +71,11 @@ def test_btype_set_props(self): finally: btype.set_properties(oldprops) - @unittest.skipIf(SKIP_BTYPES == '1', "SKIP_BTYPES is set") def test_btype_set_props_immutable(self): btype = self.client.bucket_type("pytest-maps") with self.assertRaises(RiakError): btype.set_property('datatype', 'counter') - @unittest.skipIf(SKIP_BTYPES == '1', "SKIP_BTYPES is set") def test_btype_list_buckets(self): btype = self.client.bucket_type("pytest") bucket = btype.bucket(self.bucket_name) @@ -109,7 +90,6 @@ def test_btype_list_buckets(self): self.assertIn(bucket, buckets) - @unittest.skipIf(SKIP_BTYPES == '1', "SKIP_BTYPES is set") def test_btype_list_keys(self): btype = self.client.bucket_type("pytest") bucket = btype.bucket(self.bucket_name) @@ -125,7 +105,6 @@ def test_btype_list_keys(self): self.assertIn(self.key_name, keys) - @unittest.skipIf(SKIP_BTYPES == '1', "SKIP_BTYPES is set") def test_default_btype_list_buckets(self): default_btype = self.client.bucket_type("default") bucket = default_btype.bucket(self.bucket_name) @@ -142,7 +121,6 @@ def test_default_btype_list_buckets(self): self.assertItemsEqual(buckets, self.client.get_buckets()) - @unittest.skipIf(SKIP_BTYPES == '1', "SKIP_BTYPES is set") def test_default_btype_list_keys(self): btype = self.client.bucket_type("default") bucket = btype.bucket(self.bucket_name) @@ -161,7 +139,6 @@ def test_default_btype_list_keys(self): oldapikeys = self.client.get_keys(self.client.bucket(self.bucket_name)) self.assertItemsEqual(keys, oldapikeys) - @unittest.skipIf(SKIP_BTYPES == '1', "SKIP_BTYPES is set") def test_multiget_bucket_types(self): btype = self.client.bucket_type('pytest') bucket = btype.bucket(self.bucket_name) @@ -177,7 +154,6 @@ def test_multiget_bucket_types(self): self.assertEqual(bucket, mobj.bucket) self.assertEqual(btype, mobj.bucket.bucket_type) - @unittest.skipIf(SKIP_BTYPES == '1', "SKIP_BTYPES is set") def test_write_once_bucket_type(self): btype = self.client.bucket_type('pytest-write-once') btype.set_property('write_once', True) diff --git a/riak/tests/test_client.py b/riak/tests/test_client.py new file mode 100644 index 00000000..46700a61 --- /dev/null +++ b/riak/tests/test_client.py @@ -0,0 +1,210 @@ +import platform +from six import PY2 +from threading import Thread +from riak.riak_object import RiakObject +from riak.tests import DUMMY_HTTP_PORT, DUMMY_PB_PORT, RUN_POOL +from riak.tests.base import IntegrationTestBase + +if PY2: + from Queue import Queue +else: + from queue import Queue + +if platform.python_version() < '2.7': + unittest = __import__('unittest2') +else: + import unittest + + +class ClientTests(IntegrationTestBase, unittest.TestCase): + def test_uses_client_id_if_given(self): + if self.protocol == 'pbc': + zero_client_id = "\0\0\0\0" + c = self.create_client(client_id=zero_client_id) + self.assertEqual(zero_client_id, c.client_id) + c.close() + else: + pass + + def test_request_retries(self): + # We guess at some ports that will be unused by Riak or + # anything else. + client = self.create_client(http_port=DUMMY_HTTP_PORT, + pb_port=DUMMY_PB_PORT) + + # If retries are exhausted, the final result should also be an + # error. + self.assertRaises(IOError, client.ping) + client.close() + + def test_request_retries_configurable(self): + # We guess at some ports that will be unused by Riak or + # anything else. + client = self.create_client(http_port=DUMMY_HTTP_PORT, + pb_port=DUMMY_PB_PORT) + + # Change the retry count + client.retries = 10 + self.assertEqual(10, client.retries) + + # The retry count should be a thread local + retries = Queue() + + def _target(): + retries.put(client.retries) + retries.join() + + th = Thread(target=_target) + th.start() + self.assertEqual(3, retries.get(block=True)) + retries.task_done() + th.join() + + # Modify the retries in a with statement + with client.retry_count(5): + self.assertEqual(5, client.retries) + self.assertRaises(IOError, client.ping) + client.close() + + def test_timeout_validation(self): + bucket = self.client.bucket(self.bucket_name) + key = self.key_name + obj = bucket.new(key) + for bad in [0, -1, False, "foo"]: + with self.assertRaises(ValueError): + self.client.get_buckets(timeout=bad) + + with self.assertRaises(ValueError): + for i in self.client.stream_buckets(timeout=bad): + pass + + with self.assertRaises(ValueError): + self.client.get_keys(bucket, timeout=bad) + + with self.assertRaises(ValueError): + for i in self.client.stream_keys(bucket, timeout=bad): + pass + + with self.assertRaises(ValueError): + self.client.put(obj, timeout=bad) + + with self.assertRaises(ValueError): + self.client.get(obj, timeout=bad) + + with self.assertRaises(ValueError): + self.client.delete(obj, timeout=bad) + + with self.assertRaises(ValueError): + self.client.mapred([], [], bad) + + with self.assertRaises(ValueError): + for i in self.client.stream_mapred([], [], bad): + pass + + with self.assertRaises(ValueError): + self.client.get_index(bucket, 'field1_bin', 'val1', 'val4', + timeout=bad) + + with self.assertRaises(ValueError): + for i in self.client.stream_index(bucket, 'field1_bin', 'val1', + 'val4', timeout=bad): + pass + + def test_multiget_bucket(self): + """ + Multiget operations can be invoked on buckets. + """ + keys = [self.key_name, self.randname(), self.randname()] + for key in keys: + if PY2: + self.client.bucket(self.bucket_name)\ + .new(key, encoded_data=key, content_type="text/plain")\ + .store() + else: + self.client.bucket(self.bucket_name)\ + .new(key, data=key, + content_type="text/plain").store() + results = self.client.bucket(self.bucket_name).multiget(keys) + for obj in results: + self.assertIsInstance(obj, RiakObject) + self.assertTrue(obj.exists) + if PY2: + self.assertEqual(obj.key, obj.encoded_data) + else: + self.assertEqual(obj.key, obj.data) + + def test_multiget_errors(self): + """ + Unrecoverable errors are captured along with the bucket/key + and not propagated. + """ + keys = [self.key_name, self.randname(), self.randname()] + client = self.create_client(http_port=DUMMY_HTTP_PORT, + pb_port=DUMMY_PB_PORT) + results = client.bucket(self.bucket_name).multiget(keys) + for failure in results: + self.assertIsInstance(failure, tuple) + self.assertEqual(failure[0], 'default') + self.assertEqual(failure[1], self.bucket_name) + self.assertIn(failure[2], keys) + if PY2: + self.assertIsInstance(failure[3], StandardError) # noqa + else: + self.assertIsInstance(failure[3], Exception) + client.close() + + def test_multiget_notfounds(self): + """ + Not founds work in multiget just the same as get. + """ + keys = [("default", self.bucket_name, self.key_name), + ("default", self.bucket_name, self.randname())] + results = self.client.multiget(keys) + for obj in results: + self.assertIsInstance(obj, RiakObject) + self.assertFalse(obj.exists) + + def test_multiget_pool_size(self): + """ + The pool size for multigets can be configured at client initiation + time. Multiget still works as expected. + """ + client = self.create_client(multiget_pool_size=2) + self.assertEqual(2, client._multiget_pool._size) + + keys = [self.key_name, self.randname(), self.randname()] + for key in keys: + if PY2: + client.bucket(self.bucket_name)\ + .new(key, encoded_data=key, content_type="text/plain")\ + .store() + else: + client.bucket(self.bucket_name)\ + .new(key, data=key, content_type="text/plain")\ + .store() + + results = client.bucket(self.bucket_name).multiget(keys) + for obj in results: + self.assertIsInstance(obj, RiakObject) + self.assertTrue(obj.exists) + if PY2: + self.assertEqual(obj.key, obj.encoded_data) + else: + self.assertEqual(obj.key, obj.data) + client.close() + + @unittest.skipUnless(RUN_POOL, 'RUN_POOL is 0') + def test_pool_close(self): + """ + Iterate over the connection pool and close all connections. + """ + # Do something to add to the connection pool + self.test_multiget_bucket() + if self.client.protocol == 'pbc': + self.assertGreater(len(self.client._pb_pool.resources), 1) + else: + self.assertGreater(len(self.client._http_pool.resources), 1) + # Now close them all up + self.client.close() + self.assertEqual(len(self.client._http_pool.resources), 0) + self.assertEqual(len(self.client._pb_pool.resources), 0) diff --git a/riak/tests/test_comparison.py b/riak/tests/test_comparison.py index 38a1ef9f..446bc031 100644 --- a/riak/tests/test_comparison.py +++ b/riak/tests/test_comparison.py @@ -1,25 +1,8 @@ -""" -Copyright 2015 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - +# -*- coding: utf-8 -*- import platform from riak.riak_object import RiakObject from riak.bucket import RiakBucket, BucketType -from riak.tests.test_all import BaseTestCase +from riak.tests.base import IntegrationTestBase if platform.python_version() < '2.7': unittest = __import__('unittest2') @@ -153,12 +136,14 @@ def test_object_valid_key(self): self.assertIsNone(b, 'empty object key not allowed') -class RiakClientComparisonTest(unittest.TestCase, BaseTestCase): +class RiakClientComparisonTest(IntegrationTestBase, unittest.TestCase): def test_client_eq(self): self.protocol = 'http' a = self.create_client(host='host1', http_port=11) b = self.create_client(host='host1', http_port=11) self.assertEqual(a, b) + a.close() + b.close() def test_client_nq(self): self.protocol = 'http' @@ -167,6 +152,9 @@ def test_client_nq(self): c = self.create_client(host='host1', http_port=12) self.assertNotEqual(a, b, 'matched with different hosts') self.assertNotEqual(a, c, 'matched with different ports') + a.close() + b.close() + c.close() def test_client_hash(self): self.protocol = 'http' @@ -175,6 +163,9 @@ def test_client_hash(self): c = self.create_client(host='host2', http_port=11) self.assertEqual(hash(a), hash(b), 'same object has different hashes') self.assertNotEqual(hash(a), hash(c), 'different object has same hash') + a.close() + b.close() + c.close() if __name__ == '__main__': unittest.main() diff --git a/riak/tests/test_datatypes.py b/riak/tests/test_datatypes.py index 9c6b3a0b..747de515 100644 --- a/riak/tests/test_datatypes.py +++ b/riak/tests/test_datatypes.py @@ -1,27 +1,10 @@ # -*- coding: utf-8 -*- -""" -Copyright 2015 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - import platform from riak import RiakBucket, BucketType, RiakObject import riak.datatypes as datatypes -from . import SKIP_DATATYPES -from riak.tests import test_six +from riak.tests import RUN_DATATYPES +from riak.tests.base import IntegrationTestBase +from riak.tests.comparison import Comparison if platform.python_version() < '2.7': unittest = __import__('unittest2') @@ -29,7 +12,7 @@ import unittest -class DatatypeUnitTests(object): +class DatatypeUnitTestBase(object): dtype = None bucket = RiakBucket(None, 'test', BucketType(None, 'datatypes')) @@ -67,8 +50,7 @@ def test_op_output(self): self.check_op_output(op) -class FlagUnitTests(DatatypeUnitTests, - unittest.TestCase): +class FlagUnitTests(DatatypeUnitTestBase, unittest.TestCase): dtype = datatypes.Flag def op(self, dtype): @@ -87,8 +69,7 @@ def test_disables_require_context(self): self.assertTrue(dtype.modified) -class RegisterUnitTests(DatatypeUnitTests, - unittest.TestCase): +class RegisterUnitTests(DatatypeUnitTestBase, unittest.TestCase): dtype = datatypes.Register def op(self, dtype): @@ -98,8 +79,7 @@ def check_op_output(self, op): self.assertEqual(('assign', 'foobarbaz'), op) -class CounterUnitTests(DatatypeUnitTests, - unittest.TestCase): +class CounterUnitTests(DatatypeUnitTestBase, unittest.TestCase): dtype = datatypes.Counter def op(self, dtype): @@ -109,9 +89,7 @@ def check_op_output(self, op): self.assertEqual(('increment', 5), op) -class SetUnitTests(DatatypeUnitTests, - unittest.TestCase, - test_six.Comparison): +class SetUnitTests(DatatypeUnitTestBase, unittest.TestCase, Comparison): dtype = datatypes.Set def op(self, dtype): @@ -136,8 +114,7 @@ def test_removes_require_context(self): self.assertTrue(dtype.modified) -class MapUnitTests(DatatypeUnitTests, - unittest.TestCase): +class MapUnitTests(DatatypeUnitTestBase, unittest.TestCase): dtype = datatypes.Map def op(self, dtype): @@ -170,8 +147,10 @@ def test_removes_require_context(self): self.assertTrue(dtype.modified) -class DatatypeIntegrationTests(object): - @unittest.skipIf(SKIP_DATATYPES, 'SKIP_DATATYPES is set') +@unittest.skipUnless(RUN_DATATYPES, 'RUN_DATATYPES is 0') +class DatatypeIntegrationTests(IntegrationTestBase, + unittest.TestCase, + Comparison): def test_dt_counter(self): btype = self.client.bucket_type('pytest-counters') bucket = btype.bucket(self.bucket_name) @@ -188,7 +167,6 @@ def test_dt_counter(self): mycount.reload() self.assertEqual(2, mycount.value) - @unittest.skipIf(SKIP_DATATYPES, 'SKIP_DATATYPES is set') def test_dt_set(self): btype = self.client.bucket_type('pytest-sets') bucket = btype.bucket(self.bucket_name) @@ -211,7 +189,6 @@ def test_dt_set(self): self.assertIn('Brett', myset) self.assertNotIn('Sean', myset) - @unittest.skipIf(SKIP_DATATYPES, 'SKIP_DATATYPES is set') def test_dt_map(self): btype = self.client.bucket_type('pytest-maps') bucket = btype.bucket(self.bucket_name) @@ -247,7 +224,6 @@ def test_dt_map(self): self.assertIn('f', mymap.sets) self.assertItemsEqual(['thing1', 'thing2'], mymap.sets['f'].value) - @unittest.skipIf(SKIP_DATATYPES, 'SKIP_DATATYPES is set') def test_dt_set_remove_without_context(self): btype = self.client.bucket_type('pytest-sets') bucket = btype.bucket(self.bucket_name) @@ -259,7 +235,6 @@ def test_dt_set_remove_without_context(self): with self.assertRaises(datatypes.ContextRequired): set.discard("Y") - @unittest.skipIf(SKIP_DATATYPES, 'SKIP_DATATYPES is set') def test_dt_set_remove_fetching_context(self): btype = self.client.bucket_type('pytest-sets') bucket = btype.bucket(self.bucket_name) @@ -276,7 +251,6 @@ def test_dt_set_remove_fetching_context(self): set2 = bucket.get(self.key_name) self.assertItemsEqual(['X', 'Y'], set2.value) - @unittest.skipIf(SKIP_DATATYPES, 'SKIP_DATATYPES is set') def test_dt_set_add_twice(self): btype = self.client.bucket_type('pytest-sets') bucket = btype.bucket(self.bucket_name) @@ -293,7 +267,6 @@ def test_dt_set_add_twice(self): set2 = bucket.get(self.key_name) self.assertItemsEqual(['X', 'Y'], set2.value) - @unittest.skipIf(SKIP_DATATYPES, 'SKIP_DATATYPES is set') def test_dt_set_add_wins_in_same_op(self): btype = self.client.bucket_type('pytest-sets') bucket = btype.bucket(self.bucket_name) @@ -311,7 +284,6 @@ def test_dt_set_add_wins_in_same_op(self): set2 = bucket.get(self.key_name) self.assertItemsEqual(['X', 'Y'], set2.value) - @unittest.skipIf(SKIP_DATATYPES, 'SKIP_DATATYPES is set') def test_dt_set_add_wins_in_same_op_reversed(self): btype = self.client.bucket_type('pytest-sets') bucket = btype.bucket(self.bucket_name) @@ -329,7 +301,6 @@ def test_dt_set_add_wins_in_same_op_reversed(self): set2 = bucket.get(self.key_name) self.assertItemsEqual(['X', 'Y'], set2.value) - @unittest.skipIf(SKIP_DATATYPES, 'SKIP_DATATYPES is set') def test_dt_set_remove_old_context(self): btype = self.client.bucket_type('pytest-sets') bucket = btype.bucket(self.bucket_name) @@ -351,7 +322,6 @@ def test_dt_set_remove_old_context(self): set2 = bucket.get(self.key_name) self.assertItemsEqual(['X', 'Y', 'Z'], set2.value) - @unittest.skipIf(SKIP_DATATYPES, 'SKIP_DATATYPES is set') def test_dt_set_remove_updated_context(self): btype = self.client.bucket_type('pytest-sets') bucket = btype.bucket(self.bucket_name) @@ -372,7 +342,6 @@ def test_dt_set_remove_updated_context(self): set2 = bucket.get(self.key_name) self.assertItemsEqual(['X', 'Y'], set2.value) - @unittest.skipIf(SKIP_DATATYPES, 'SKIP_DATATYPES is set') def test_dt_map_remove_set_update_same_op(self): btype = self.client.bucket_type('pytest-maps') bucket = btype.bucket(self.bucket_name) @@ -390,7 +359,6 @@ def test_dt_map_remove_set_update_same_op(self): map2 = bucket.get(self.key_name) self.assertItemsEqual(["Z"], map2.sets['set']) - @unittest.skipIf(SKIP_DATATYPES, 'SKIP_DATATYPES is set') def test_dt_map_remove_counter_increment_same_op(self): btype = self.client.bucket_type('pytest-maps') bucket = btype.bucket(self.bucket_name) @@ -408,7 +376,6 @@ def test_dt_map_remove_counter_increment_same_op(self): map2 = bucket.get(self.key_name) self.assertEqual(2, map2.counters['counter'].value) - @unittest.skipIf(SKIP_DATATYPES, 'SKIP_DATATYPES is set') def test_dt_map_remove_map_update_same_op(self): btype = self.client.bucket_type('pytest-maps') bucket = btype.bucket(self.bucket_name) @@ -426,7 +393,6 @@ def test_dt_map_remove_map_update_same_op(self): map2 = bucket.get(self.key_name) self.assertItemsEqual(["Z"], map2.maps['map'].sets['set']) - @unittest.skipIf(SKIP_DATATYPES, 'SKIP_DATATYPES is set') def test_dt_set_return_body_true_default(self): btype = self.client.bucket_type('pytest-sets') bucket = btype.bucket(self.bucket_name) @@ -444,7 +410,6 @@ def test_dt_set_return_body_true_default(self): myset.store() self.assertItemsEqual(myset.value, ['Y']) - @unittest.skipIf(SKIP_DATATYPES, 'SKIP_DATATYPES is set') def test_dt_map_return_body_true_default(self): btype = self.client.bucket_type('pytest-maps') bucket = btype.bucket(self.bucket_name) @@ -469,7 +434,6 @@ def test_dt_map_return_body_true_default(self): self.assertEqual(mymap.value, {}) - @unittest.skipIf(SKIP_DATATYPES, 'SKIP_DATATYPES is set') def test_delete_datatype(self): ctype = self.client.bucket_type('pytest-counters') cbucket = ctype.bucket(self.bucket_name) diff --git a/riak/tests/test_feature_detection.py b/riak/tests/test_feature_detection.py index 682c5ac2..d88334aa 100644 --- a/riak/tests/test_feature_detection.py +++ b/riak/tests/test_feature_detection.py @@ -1,21 +1,4 @@ -""" -Copyright 2012-2015 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - +# -*- coding: utf-8 -*- import platform from riak.transports.feature_detect import FeatureDetection diff --git a/riak/tests/test_filters.py b/riak/tests/test_filters.py index 00e9d0af..c821ce95 100644 --- a/riak/tests/test_filters.py +++ b/riak/tests/test_filters.py @@ -1,21 +1,4 @@ -""" -Copyright 2015 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - +# -*- coding: utf-8 -*- import platform from riak.mapreduce import RiakKeyFilter from riak import key_filter diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index d1b28298..c9eb94dd 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -1,22 +1,4 @@ # -*- coding: utf-8 -*- -""" -Copyright 2015 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - import os import platform from six import string_types, PY2, PY3 @@ -25,7 +7,9 @@ from time import sleep from riak import ConflictError, RiakBucket, RiakError from riak.resolver import default_resolver, last_written_resolver -from . import SKIP_RESOLVE +from riak.tests import RUN_RESOLVE +from riak.tests.base import IntegrationTestBase +from riak.tests.comparison import Comparison try: import simplejson as json @@ -47,6 +31,23 @@ test_pickle_loads = pickle.loads +testrun_sibs_bucket = 'sibsbucket' +testrun_props_bucket = 'propsbucket' + + +def setUpModule(): + c = IntegrationTestBase.create_client() + c.bucket(testrun_sibs_bucket).allow_mult = True + c.close() + + +def tearDownModule(): + c = IntegrationTestBase.create_client() + c.bucket(testrun_sibs_bucket).clear_properties() + c.bucket(testrun_props_bucket).clear_properties() + c.close() + + class NotJsonSerializable(object): def __init__(self, *args, **kwargs): @@ -71,7 +72,23 @@ def __eq__(self, other): return True -class BasicKVTests(object): +class BasicKVTests(IntegrationTestBase, unittest.TestCase, Comparison): + def test_no_returnbody(self): + bucket = self.client.bucket(self.bucket_name) + o = bucket.new(self.key_name, "bar").store(return_body=False) + self.assertEqual(o.vclock, None) + + def test_many_link_headers_should_work_fine(self): + bucket = self.client.bucket(self.bucket_name) + o = bucket.new("lots_of_links", "My god, it's full of links!") + for i in range(0, 300): + link = ("other", "key%d" % i, "next") + o.add_link(link) + + o.store() + stored_object = bucket.get("lots_of_links") + self.assertEqual(len(stored_object.links), 300) + def test_is_alive(self): self.assertTrue(self.client.is_alive()) @@ -340,22 +357,28 @@ def test_bucket_delete(self): self.assertFalse(obj.exists) def test_set_bucket_properties(self): - bucket = self.client.bucket(self.props_bucket) + bucket = self.client.bucket(testrun_props_bucket) # Test setting allow mult... bucket.allow_mult = True # Test setting nval... bucket.n_val = 1 - bucket2 = self.create_client().bucket(self.props_bucket) + c2 = self.create_client() + bucket2 = c2.bucket(testrun_props_bucket) self.assertTrue(bucket2.allow_mult) self.assertEqual(bucket2.n_val, 1) # Test setting multiple properties... bucket.set_properties({"allow_mult": False, "n_val": 2}) - bucket3 = self.create_client().bucket(self.props_bucket) + c3 = self.create_client() + bucket3 = c3.bucket(testrun_props_bucket) self.assertFalse(bucket3.allow_mult) self.assertEqual(bucket3.n_val, 2) + # clean up! + c2.close() + c3.close() + def test_if_none_match(self): bucket = self.client.bucket(self.bucket_name) obj = bucket.get(self.key_name) @@ -373,7 +396,7 @@ def test_if_none_match(self): def test_siblings(self): # Set up the bucket, clear any existing object... - bucket = self.client.bucket(self.sibs_bucket) + bucket = self.client.bucket(testrun_sibs_bucket) obj = bucket.get(self.key_name) bucket.allow_mult = True @@ -409,10 +432,9 @@ def test_siblings(self): self.assertEqual(len(obj.siblings), 1) self.assertEqual(obj.data, resolved_sibling.data) - @unittest.skipIf(SKIP_RESOLVE == '1', - "skip requested for resolvers test") + @unittest.skipUnless(RUN_RESOLVE, "RUN_RESOLVE is 0") def test_resolution(self): - bucket = self.client.bucket(self.sibs_bucket) + bucket = self.client.bucket(testrun_sibs_bucket) obj = bucket.get(self.key_name) bucket.allow_mult = True @@ -466,17 +488,16 @@ def max_value_resolver(obj): self.assertEqual(bucket.resolver, default_resolver) # reset self.assertEqual(self.client.resolver, default_resolver) # reset - @unittest.skipIf(SKIP_RESOLVE == '1', - "skip requested for resolvers test") + @unittest.skipUnless(RUN_RESOLVE, "RUN_RESOLVE is 0") def test_resolution_default(self): # If no resolver is setup, be sure to resolve to default_resolver - bucket = self.client.bucket(self.sibs_bucket) + bucket = self.client.bucket(testrun_sibs_bucket) self.assertEqual(self.client.resolver, default_resolver) self.assertEqual(bucket.resolver, default_resolver) def test_tombstone_siblings(self): # Set up the bucket, clear any existing object... - bucket = self.client.bucket(self.sibs_bucket) + bucket = self.client.bucket(testrun_sibs_bucket) obj = bucket.get(self.key_name) bucket.allow_mult = True @@ -610,9 +631,9 @@ def generate_siblings(self, original, count=5, delay=None): return vals -class BucketPropsTest(object): +class BucketPropsTest(IntegrationTestBase, unittest.TestCase): def test_rw_settings(self): - bucket = self.client.bucket(self.props_bucket) + bucket = self.client.bucket(testrun_props_bucket) self.assertEqual(bucket.r, "quorum") self.assertEqual(bucket.w, "quorum") self.assertEqual(bucket.dw, "quorum") @@ -637,7 +658,7 @@ def test_rw_settings(self): bucket.clear_properties() def test_primary_quora(self): - bucket = self.client.bucket(self.props_bucket) + bucket = self.client.bucket(testrun_props_bucket) self.assertEqual(bucket.pr, 0) self.assertEqual(bucket.pw, 0) @@ -651,7 +672,7 @@ def test_primary_quora(self): bucket.clear_properties() def test_clear_bucket_properties(self): - bucket = self.client.bucket(self.props_bucket) + bucket = self.client.bucket(testrun_props_bucket) bucket.allow_mult = True self.assertTrue(bucket.allow_mult) bucket.n_val = 1 @@ -663,20 +684,20 @@ def test_clear_bucket_properties(self): self.assertEqual(bucket.n_val, 3) -class KVFileTests(object): +class KVFileTests(IntegrationTestBase, unittest.TestCase): def test_store_binary_object_from_file(self): bucket = self.client.bucket(self.bucket_name) - filepath = os.path.join(os.path.dirname(__file__), 'test_all.py') - obj = bucket.new_from_file(self.key_name, filepath) + obj = bucket.new_from_file(self.key_name, __file__) obj.store() obj = bucket.get(self.key_name) self.assertNotEqual(obj.encoded_data, None) - self.assertEqual(obj.content_type, "text/x-python") + self.assertTrue(obj.content_type == 'text/x-python' or + obj.content_type == 'application/x-python-code') def test_store_binary_object_from_file_should_use_default_mimetype(self): bucket = self.client.bucket(self.bucket_name) filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), - os.pardir, os.pardir, 'THANKS') + os.pardir, os.pardir, 'README.rst') obj = bucket.new_from_file(self.key_name, filepath) obj.store() obj = bucket.get(self.key_name) @@ -691,7 +712,7 @@ def test_store_binary_object_from_file_should_fail_if_file_not_found(self): self.assertFalse(obj.exists) -class CounterTests(object): +class CounterTests(IntegrationTestBase, unittest.TestCase): def test_counter_requires_allow_mult(self): bucket = self.client.bucket(self.bucket_name) if bucket.allow_mult: @@ -702,7 +723,7 @@ def test_counter_requires_allow_mult(self): bucket.update_counter(self.key_name, 10) def test_counter_ops(self): - bucket = self.client.bucket(self.sibs_bucket) + bucket = self.client.bucket(testrun_sibs_bucket) self.assertTrue(bucket.allow_mult) # Non-existent counter has no value diff --git a/riak/tests/test_mapreduce.py b/riak/tests/test_mapreduce.py index c15ff7b1..b6cd068f 100644 --- a/riak/tests/test_mapreduce.py +++ b/riak/tests/test_mapreduce.py @@ -1,38 +1,37 @@ # -*- coding: utf-8 -*- -""" -Copyright 2015 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 +from __future__ import print_function -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" +import platform -from __future__ import print_function from six import PY2 from riak.mapreduce import RiakMapReduce from riak import key_filter, RiakError +from riak.tests import RUN_YZ +from riak.tests.base import IntegrationTestBase from riak.tests.test_yokozuna import wait_for_yz_index from riak.tests import RUN_SECURITY -import platform +from riak.tests.yz_setup import yzSetUp, yzTearDown -from . import RUN_YZ if platform.python_version() < '2.7': unittest = __import__('unittest2') else: import unittest -class LinkTests(object): +testrun_yz_mr = {'btype': 'pytest-mr', + 'bucket': 'mrbucket', + 'index': 'mrbucket'} + + +def setUpModule(): + yzSetUp(testrun_yz_mr) + + +def tearDownModule(): + yzTearDown(testrun_yz_mr) + + +class LinkTests(IntegrationTestBase, unittest.TestCase): def test_store_and_get_links(self): # Create the object... bucket = self.client.bucket(self.bucket_name) @@ -98,7 +97,7 @@ def test_link_walking(self): self.assertEqual(len(results), 1) -class ErlangMapReduceTests(object): +class ErlangMapReduceTests(IntegrationTestBase, unittest.TestCase): def test_erlang_map_reduce(self): # Create the object... bucket = self.client.bucket(self.bucket_name) @@ -204,7 +203,8 @@ def test_client_exceptional_paths(self): mr.add_key_filter("tokenize", "-", 1) -class JSMapReduceTests(object): +class JSMapReduceTests(IntegrationTestBase, unittest.TestCase): + def test_javascript_source_map(self): # Create the object... bucket = self.client.bucket(self.bucket_name) @@ -520,13 +520,13 @@ def test_mr_list_add_mix(self): u'"fooval2"', u'"fooval3"']) - @unittest.skipUnless(RUN_YZ, 'RUN_YZ is undefined') + @unittest.skipUnless(RUN_YZ, 'RUN_YZ is 0') def test_mr_search(self): """ Try a successful map/reduce from search results. """ - btype = self.client.bucket_type(self.yz_mr['btype']) - bucket = btype.bucket(self.yz_mr['bucket']) + btype = self.client.bucket_type(testrun_yz_mr['btype']) + bucket = btype.bucket(testrun_yz_mr['bucket']) bucket.new("Pebbles", {"name_s": "Fruity Pebbles", "maker_s": "Post", "sugar_i": 9, @@ -554,7 +554,7 @@ def test_mr_search(self): "fruit_b": False}).store() # Wait for Solr to catch up wait_for_yz_index(bucket, "Crunch") - mr = RiakMapReduce(self.client).search(self.yz_mr['bucket'], + mr = RiakMapReduce(self.client).search(testrun_yz_mr['bucket'], 'fruit_b:false') mr.map("""function(v) { var solr_doc = JSON.parse(v.values[0].data); @@ -564,7 +564,7 @@ def test_mr_search(self): self.assertEqual(result, [100]) -class MapReduceAliasTests(object): +class MapReduceAliasTests(IntegrationTestBase, unittest.TestCase): """This tests the map reduce aliases""" def test_map_values(self): @@ -759,7 +759,7 @@ def test_filter_not_found(self): self.assertEqual(sorted(result), [1, 2]) -class MapReduceStreamTests(object): +class MapReduceStreamTests(IntegrationTestBase, unittest.TestCase): def test_stream_results(self): bucket = self.client.bucket(self.bucket_name) bucket.new('one', data=1).store() diff --git a/riak/tests/test_pool.py b/riak/tests/test_pool.py index 6355eee0..f1088244 100644 --- a/riak/tests/test_pool.py +++ b/riak/tests/test_pool.py @@ -1,29 +1,12 @@ -""" -Copyright 2015 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - +# -*- coding: utf-8 -*- from six import PY2 import platform from threading import Thread, currentThread from riak.transports.pool import Pool, BadResource from random import SystemRandom from time import sleep -from . import SKIP_POOL -from riak.tests import test_six +from riak.tests import RUN_POOL +from riak.tests.comparison import Comparison if platform.python_version() < '2.7': unittest = __import__('unittest2') @@ -54,10 +37,8 @@ def create_resource(self): return [] -@unittest.skipIf(SKIP_POOL, - 'Skipping connection pool tests') -class PoolTest(unittest.TestCase, - test_six.Comparison): +@unittest.skipUnless(RUN_POOL, 'RUN_POOL is 0') +class PoolTest(unittest.TestCase, Comparison): def test_yields_new_object_when_empty(self): """ diff --git a/riak/tests/test_search.py b/riak/tests/test_search.py index eed22e2c..7cc369b6 100644 --- a/riak/tests/test_search.py +++ b/riak/tests/test_search.py @@ -1,157 +1,158 @@ # -*- coding: utf-8 -*- -""" -Copyright 2015 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - from __future__ import print_function import platform -from . import SKIP_SEARCH +from riak.tests import RUN_SEARCH, RUN_YZ +from riak.tests.base import IntegrationTestBase + if platform.python_version() < '2.7': unittest = __import__('unittest2') else: import unittest +testrun_search_bucket = 'searchbucket' + + +def setUpModule(): + if RUN_SEARCH and not RUN_YZ: + c = IntegrationTestBase.create_client() + b = c.bucket(testrun_search_bucket) + b.enable_search() + c.close() + + +def tearDownModule(): + if RUN_SEARCH and not RUN_YZ: + c = IntegrationTestBase.create_client() + b = c.bucket(testrun_search_bucket) + b.clear_properties() + c.close() -class EnableSearchTests(object): - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') + +@unittest.skipUnless(RUN_SEARCH, 'RUN_SEARCH is 0') +class EnableSearchTests(IntegrationTestBase, unittest.TestCase): def test_bucket_search_enabled(self): bucket = self.client.bucket(self.bucket_name) self.assertFalse(bucket.search_enabled()) - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_enable_search_commit_hook(self): - bucket = self.client.bucket(self.search_bucket) + bucket = self.client.bucket(testrun_search_bucket) bucket.clear_properties() - self.assertFalse(self.create_client(). - bucket(self.search_bucket). - search_enabled()) + + c = self.create_client() + self.assertFalse(c.bucket(testrun_search_bucket).search_enabled()) + c.close() + bucket.enable_search() - self.assertTrue(self.create_client(). - bucket(self.search_bucket). - search_enabled()) - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') + c = self.create_client() + self.assertTrue(c.bucket(testrun_search_bucket).search_enabled()) + c.close() + def test_disable_search_commit_hook(self): - bucket = self.client.bucket(self.search_bucket) + bucket = self.client.bucket(testrun_search_bucket) bucket.clear_properties() bucket.enable_search() - self.assertTrue(self.create_client().bucket(self.search_bucket) - .search_enabled()) + + c = self.create_client() + self.assertTrue(c.bucket(testrun_search_bucket).search_enabled()) + c.close() + bucket.disable_search() - self.assertFalse(self.create_client().bucket(self.search_bucket) - .search_enabled()) + + c = self.create_client() + self.assertFalse(c.bucket(testrun_search_bucket).search_enabled()) + c.close() + bucket.enable_search() -class SolrSearchTests(object): - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') +@unittest.skipUnless(RUN_SEARCH, 'RUN_SEARCH is 0') +class SolrSearchTests(IntegrationTestBase, unittest.TestCase): def test_add_document_to_index(self): - self.client.fulltext_add(self.search_bucket, + self.client.fulltext_add(testrun_search_bucket, [{"id": "doc", "username": "tony"}]) - results = self.client.fulltext_search(self.search_bucket, + results = self.client.fulltext_search(testrun_search_bucket, "username:tony") self.assertEqual("tony", results['docs'][0]['username']) - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_add_multiple_documents_to_index(self): self.client.fulltext_add( - self.search_bucket, + testrun_search_bucket, [{"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}]) results = self.client.fulltext_search( - self.search_bucket, "username:russell OR username:dizzy") + testrun_search_bucket, "username:russell OR username:dizzy") self.assertEqual(2, len(results['docs'])) - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_id(self): self.client.fulltext_add( - self.search_bucket, + testrun_search_bucket, [{"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}]) - self.client.fulltext_delete(self.search_bucket, docs=["dizzy"]) + self.client.fulltext_delete(testrun_search_bucket, docs=["dizzy"]) results = self.client.fulltext_search( - self.search_bucket, "username:russell OR username:dizzy") + testrun_search_bucket, "username:russell OR username:dizzy") self.assertEqual(1, len(results['docs'])) - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_query(self): self.client.fulltext_add( - self.search_bucket, + testrun_search_bucket, [{"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}]) self.client.fulltext_delete( - self.search_bucket, + testrun_search_bucket, queries=["username:dizzy", "username:russell"]) results = self.client.fulltext_search( - self.search_bucket, "username:russell OR username:dizzy") + testrun_search_bucket, "username:russell OR username:dizzy") self.assertEqual(0, len(results['docs'])) - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_query_and_id(self): self.client.fulltext_add( - self.search_bucket, + testrun_search_bucket, [{"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}]) self.client.fulltext_delete( - self.search_bucket, + testrun_search_bucket, docs=["dizzy"], queries=["username:russell"]) results = self.client.fulltext_search( - self.search_bucket, + testrun_search_bucket, "username:russell OR username:dizzy") self.assertEqual(0, len(results['docs'])) -class SearchTests(object): - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') +@unittest.skipUnless(RUN_SEARCH, 'RUN_SEARCH is 0') +class SearchTests(IntegrationTestBase, unittest.TestCase): def test_solr_search_from_bucket(self): - bucket = self.client.bucket(self.search_bucket) + bucket = self.client.bucket(testrun_search_bucket) bucket.new("user", {"username": "roidrage"}).store() results = bucket.search("username:roidrage") self.assertEqual(1, len(results['docs'])) - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_solr_search_with_params_from_bucket(self): - bucket = self.client.bucket(self.search_bucket) + bucket = self.client.bucket(testrun_search_bucket) bucket.new("user", {"username": "roidrage"}).store() results = bucket.search("username:roidrage", wt="xml") self.assertEqual(1, len(results['docs'])) - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_solr_search_with_params(self): - bucket = self.client.bucket(self.search_bucket) + bucket = self.client.bucket(testrun_search_bucket) bucket.new("user", {"username": "roidrage"}).store() results = self.client.fulltext_search( - self.search_bucket, + testrun_search_bucket, "username:roidrage", wt="xml") self.assertEqual(1, len(results['docs'])) - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_solr_search(self): - bucket = self.client.bucket(self.search_bucket) + bucket = self.client.bucket(testrun_search_bucket) bucket.new("user", {"username": "roidrage"}).store() - results = self.client.fulltext_search(self.search_bucket, + results = self.client.fulltext_search(testrun_search_bucket, "username:roidrage") self.assertEqual(1, len(results["docs"])) - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_search_integration(self): # Create some objects to search across... - bucket = self.client.bucket(self.search_bucket) + bucket = self.client.bucket(testrun_search_bucket) bucket.new("one", {"foo": "one", "bar": "red"}).store() bucket.new("two", {"foo": "two", "bar": "green"}).store() bucket.new("three", {"foo": "three", "bar": "blue"}).store() @@ -159,7 +160,7 @@ def test_search_integration(self): bucket.new("five", {"foo": "five", "bar": "yellow"}).store() # Run some operations... - results = self.client.fulltext_search(self.search_bucket, + results = self.client.fulltext_search(testrun_search_bucket, "foo:one OR foo:two") if (len(results) == 0): print("\n\nNot running test \"testSearchIntegration()\".\n") @@ -170,6 +171,6 @@ def test_search_integration(self): self.assertEqual(len(results['docs']), 2) query = "(foo:one OR foo:two OR foo:three OR foo:four) AND\ (NOT bar:green)" - results = self.client.fulltext_search(self.search_bucket, query) + results = self.client.fulltext_search(testrun_search_bucket, query) self.assertEqual(len(results['docs']), 3) diff --git a/riak/tests/test_security.py b/riak/tests/test_security.py index f0489039..85588ee0 100644 --- a/riak/tests/test_security.py +++ b/riak/tests/test_security.py @@ -1,45 +1,40 @@ # -*- coding: utf-8 -*- -""" -Copyright 2015 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - +import platform import sys + from riak.tests import RUN_SECURITY, SECURITY_USER, SECURITY_PASSWD, \ SECURITY_CACERT, SECURITY_KEY, SECURITY_CERT, SECURITY_REVOKED, \ SECURITY_CERT_USER, SECURITY_CERT_PASSWD, SECURITY_BAD_CERT, \ - SECURITY_CREDS, SECURITY_CIPHERS + SECURITY_CIPHERS from riak.security import SecurityCreds -if sys.version_info < (2, 7): +from riak.tests.base import IntegrationTestBase + +if platform.python_version() < '2.7': unittest = __import__('unittest2') else: import unittest -class SecurityTests(object): - @unittest.skipIf(RUN_SECURITY, 'RUN_SECURITY is set') +class SecurityTests(IntegrationTestBase, unittest.TestCase): + @unittest.skipIf(RUN_SECURITY, 'RUN_SECURITY is 1') def test_security_disabled(self): - client = self.create_client(credentials=SECURITY_CREDS) + """ + Test valid security settings without security enabled + """ + topts = {'timeout': 1} + # NB: can't use SECURITY_CREDS here since they won't be set + # if RUN_SECURITY is UN-set + creds = SecurityCreds(username='foo', password='bar') + client = self.create_client(credentials=creds, + transport_options=topts) myBucket = client.bucket('test') val1 = "foobar" key1 = myBucket.new('x', data=val1) with self.assertRaises(Exception): key1.store() + client.close() - @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') + @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is 0') def test_security_basic_connection(self): myBucket = self.client.bucket('test') val1 = "foobar" @@ -47,7 +42,7 @@ def test_security_basic_connection(self): key1.store() myBucket.get('x') - @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') + @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is 0') def test_security_bad_user(self): creds = SecurityCreds(username='foo', password=SECURITY_PASSWD, @@ -56,8 +51,9 @@ def test_security_bad_user(self): client = self.create_client(credentials=creds) with self.assertRaises(Exception): client.get_buckets() + client.close() - @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') + @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is 0') def test_security_bad_password(self): creds = SecurityCreds(username=SECURITY_USER, password='foo', @@ -66,8 +62,9 @@ def test_security_bad_password(self): client = self.create_client(credentials=creds) with self.assertRaises(Exception): client.get_buckets() + client.close() - @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') + @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is 0') def test_security_invalid_cert(self): creds = SecurityCreds(username=SECURITY_USER, password=SECURITY_PASSWD, @@ -76,8 +73,9 @@ def test_security_invalid_cert(self): client = self.create_client(credentials=creds) with self.assertRaises(Exception): client.get_buckets() + client.close() - @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') + @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is 0') def test_security_password_without_cacert(self): creds = SecurityCreds(username=SECURITY_USER, password=SECURITY_PASSWD, @@ -88,8 +86,9 @@ def test_security_password_without_cacert(self): val1 = "foobar" key1 = myBucket.new('x', data=val1) key1.store() + client.close() - @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') + @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is 0') def test_security_cert_authentication(self): creds = SecurityCreds(username=SECURITY_CERT_USER, password=SECURITY_CERT_PASSWD, @@ -110,10 +109,12 @@ def test_security_cert_authentication(self): with self.assertRaises(Exception): key1.store() myBucket.get('x') + client.close() - @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') + @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is 0') def test_security_revoked_cert(self): - creds = SecurityCreds(username=SECURITY_USER, password=SECURITY_PASSWD, + creds = SecurityCreds(username=SECURITY_USER, + password=SECURITY_PASSWD, ciphers=SECURITY_CIPHERS, cacert_file=SECURITY_CACERT, crl_file=SECURITY_REVOKED) @@ -124,8 +125,9 @@ def test_security_revoked_cert(self): client = self.create_client(credentials=creds) with self.assertRaises(Exception): client.get_buckets() + client.close() - @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') + @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is 0') def test_security_bad_ca_cert(self): creds = SecurityCreds(username=SECURITY_USER, password=SECURITY_PASSWD, ciphers=SECURITY_CIPHERS, @@ -133,8 +135,9 @@ def test_security_bad_ca_cert(self): client = self.create_client(credentials=creds) with self.assertRaises(Exception): client.get_buckets() + client.close() - @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') + @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is 0') def test_security_ciphers(self): creds = SecurityCreds(username=SECURITY_USER, password=SECURITY_PASSWD, ciphers=SECURITY_CIPHERS, @@ -145,8 +148,9 @@ def test_security_ciphers(self): key1 = myBucket.new('x', data=val1) key1.store() myBucket.get('x') + client.close() - @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') + @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is 0') def test_security_bad_ciphers(self): creds = SecurityCreds(username=SECURITY_USER, password=SECURITY_PASSWD, cacert_file=SECURITY_CACERT, @@ -154,3 +158,4 @@ def test_security_bad_ciphers(self): client = self.create_client(credentials=creds) with self.assertRaises(Exception): client.get_buckets() + client.close() diff --git a/riak/tests/test_timeseries.py b/riak/tests/test_timeseries.py new file mode 100644 index 00000000..8835e21d --- /dev/null +++ b/riak/tests/test_timeseries.py @@ -0,0 +1,299 @@ +# -*- coding: utf-8 -*- +import datetime +import platform +import riak_pb + +from riak import RiakError +from riak.table import Table +from riak.ts_object import TsObject +from riak.transports.pbc.codec import RiakPbcCodec +from riak.util import str_to_bytes, bytes_to_str +from riak.tests import RUN_TIMESERIES +from riak.tests.base import IntegrationTestBase + +if platform.python_version() < '2.7': + unittest = __import__('unittest2') +else: + import unittest + +table_name = 'GeoCheckin' + +bd0 = '时间序列' +bd1 = 'временные ряды' + +fiveMins = datetime.timedelta(0, 300) +ts0 = datetime.datetime(2015, 1, 1, 12, 0, 0) +ts1 = ts0 + fiveMins + + +@unittest.skipUnless(RUN_TIMESERIES, 'RUN_TIMESERIES is 0') +class TimeseriesUnitTests(unittest.TestCase): + def setUp(self): + self.c = RiakPbcCodec() + self.ts0ms = self.c._unix_time_millis(ts0) + self.ts1ms = self.c._unix_time_millis(ts1) + self.rows = [ + [bd0, 0, 1.2, ts0, True], + [bd1, 3, 4.5, ts1, False] + ] + self.test_key = ['hash1', 'user2', ts0] + self.table = Table(None, 'test-table') + + def validate_keyreq(self, req): + self.assertEqual(self.table.name, bytes_to_str(req.table)) + self.assertEqual(len(self.test_key), len(req.key)) + self.assertEqual('hash1', bytes_to_str(req.key[0].varchar_value)) + self.assertEqual('user2', bytes_to_str(req.key[1].varchar_value)) + self.assertEqual(self.ts0ms, req.key[2].timestamp_value) + + def test_encode_data_for_get(self): + req = riak_pb.TsGetReq() + self.c._encode_timeseries_keyreq(self.table, self.test_key, req) + self.validate_keyreq(req) + + def test_encode_data_for_delete(self): + req = riak_pb.TsDelReq() + self.c._encode_timeseries_keyreq(self.table, self.test_key, req) + self.validate_keyreq(req) + + def test_encode_data_for_put(self): + tsobj = TsObject(None, self.table, self.rows, None) + ts_put_req = riak_pb.TsPutReq() + self.c._encode_timeseries_put(tsobj, ts_put_req) + + # NB: expected, actual + self.assertEqual(self.table.name, bytes_to_str(ts_put_req.table)) + self.assertEqual(len(self.rows), len(ts_put_req.rows)) + + r0 = ts_put_req.rows[0] + self.assertEqual(bytes_to_str(r0.cells[0].varchar_value), + self.rows[0][0]) + self.assertEqual(r0.cells[1].sint64_value, self.rows[0][1]) + self.assertEqual(r0.cells[2].double_value, self.rows[0][2]) + self.assertEqual(r0.cells[3].timestamp_value, self.ts0ms) + self.assertEqual(r0.cells[4].boolean_value, self.rows[0][4]) + + r1 = ts_put_req.rows[1] + self.assertEqual(bytes_to_str(r1.cells[0].varchar_value), + self.rows[1][0]) + self.assertEqual(r1.cells[1].sint64_value, self.rows[1][1]) + self.assertEqual(r1.cells[2].double_value, self.rows[1][2]) + self.assertEqual(r1.cells[3].timestamp_value, self.ts1ms) + self.assertEqual(r1.cells[4].boolean_value, self.rows[1][4]) + + def test_encode_data_for_listkeys(self): + req = riak_pb.TsListKeysReq() + self.c._encode_timeseries_listkeysreq(self.table, req, 1234) + self.assertEqual(self.table.name, bytes_to_str(req.table)) + self.assertEqual(1234, req.timeout) + + def test_decode_data_from_query(self): + tqr = riak_pb.TsQueryResp() + + c0 = tqr.columns.add() + c0.name = str_to_bytes('col_varchar') + c0.type = riak_pb.TsColumnType.Value('VARCHAR') + c1 = tqr.columns.add() + c1.name = str_to_bytes('col_integer') + c1.type = riak_pb.TsColumnType.Value('SINT64') + c2 = tqr.columns.add() + c2.name = str_to_bytes('col_double') + c2.type = riak_pb.TsColumnType.Value('DOUBLE') + c3 = tqr.columns.add() + c3.name = str_to_bytes('col_timestamp') + c3.type = riak_pb.TsColumnType.Value('TIMESTAMP') + c4 = tqr.columns.add() + c4.name = str_to_bytes('col_boolean') + c4.type = riak_pb.TsColumnType.Value('BOOLEAN') + + r0 = tqr.rows.add() + r0c0 = r0.cells.add() + r0c0.varchar_value = str_to_bytes(self.rows[0][0]) + r0c1 = r0.cells.add() + r0c1.sint64_value = self.rows[0][1] + r0c2 = r0.cells.add() + r0c2.double_value = self.rows[0][2] + r0c3 = r0.cells.add() + r0c3.timestamp_value = self.ts0ms + r0c4 = r0.cells.add() + r0c4.boolean_value = self.rows[0][4] + + r1 = tqr.rows.add() + r1c0 = r1.cells.add() + r1c0.varchar_value = str_to_bytes(self.rows[1][0]) + r1c1 = r1.cells.add() + r1c1.sint64_value = self.rows[1][1] + r1c2 = r1.cells.add() + r1c2.double_value = self.rows[1][2] + r1c3 = r1.cells.add() + r1c3.timestamp_value = self.ts1ms + r1c4 = r1.cells.add() + r1c4.boolean_value = self.rows[1][4] + + tsobj = TsObject(None, self.table, [], []) + c = RiakPbcCodec() + c._decode_timeseries(tqr, tsobj) + + self.assertEqual(len(self.rows), len(tsobj.rows)) + self.assertEqual(len(tqr.columns), len(tsobj.columns)) + + c = tsobj.columns + self.assertEqual(c[0][0], 'col_varchar') + self.assertEqual(c[0][1], riak_pb.TsColumnType.Value('VARCHAR')) + self.assertEqual(c[1][0], 'col_integer') + self.assertEqual(c[1][1], riak_pb.TsColumnType.Value('SINT64')) + self.assertEqual(c[2][0], 'col_double') + self.assertEqual(c[2][1], riak_pb.TsColumnType.Value('DOUBLE')) + self.assertEqual(c[3][0], 'col_timestamp') + self.assertEqual(c[3][1], riak_pb.TsColumnType.Value('TIMESTAMP')) + self.assertEqual(c[4][0], 'col_boolean') + self.assertEqual(c[4][1], riak_pb.TsColumnType.Value('BOOLEAN')) + + r0 = tsobj.rows[0] + self.assertEqual(r0[0], self.rows[0][0]) + self.assertEqual(r0[1], self.rows[0][1]) + self.assertEqual(r0[2], self.rows[0][2]) + self.assertEqual(r0[3], ts0) + self.assertEqual(r0[4], self.rows[0][4]) + + r1 = tsobj.rows[1] + self.assertEqual(r1[0], self.rows[1][0]) + self.assertEqual(r1[1], self.rows[1][1]) + self.assertEqual(r1[2], self.rows[1][2]) + self.assertEqual(r1[3], ts1) + self.assertEqual(r1[4], self.rows[1][4]) + + +@unittest.skipUnless(RUN_TIMESERIES, 'RUN_TIMESERIES is 0') +class TimeseriesTests(IntegrationTestBase, unittest.TestCase): + @classmethod + def setUpClass(cls): + super(TimeseriesTests, cls).setUpClass() + cls.now = datetime.datetime.utcfromtimestamp(144379690) + fiveMinsAgo = cls.now - fiveMins + tenMinsAgo = fiveMinsAgo - fiveMins + fifteenMinsAgo = tenMinsAgo - fiveMins + twentyMinsAgo = fifteenMinsAgo - fiveMins + twentyFiveMinsAgo = twentyMinsAgo - fiveMins + + client = cls.create_client() + table = client.table(table_name) + rows = [ + ['hash1', 'user2', twentyFiveMinsAgo, 'typhoon', 90.3], + ['hash1', 'user2', twentyMinsAgo, 'hurricane', 82.3], + ['hash1', 'user2', fifteenMinsAgo, 'rain', 79.0], + ['hash1', 'user2', fiveMinsAgo, 'wind', None], + ['hash1', 'user2', cls.now, 'snow', 20.1] + ] + ts_obj = table.new(rows) + result = ts_obj.store() + if not result: + raise AssertionError("expected success") + client.close() + + codec = RiakPbcCodec() + cls.nowMsec = codec._unix_time_millis(cls.now) + cls.fiveMinsAgo = fiveMinsAgo + cls.twentyMinsAgo = twentyMinsAgo + cls.twentyFiveMinsAgo = twentyFiveMinsAgo + cls.tenMinsAgoMsec = codec._unix_time_millis(tenMinsAgo) + cls.twentyMinsAgoMsec = codec._unix_time_millis(twentyMinsAgo) + cls.numCols = len(rows[0]) + cls.rows = rows + + def validate_data(self, ts_obj): + if ts_obj.columns is not None: + self.assertEqual(len(ts_obj.columns), self.numCols) + self.assertEqual(len(ts_obj.rows), 1) + row = ts_obj.rows[0] + self.assertEqual(row[0], 'hash1') + self.assertEqual(row[1], 'user2') + self.assertEqual(row[2], self.fiveMinsAgo) + self.assertEqual(row[3], 'wind') + self.assertIsNone(row[4]) + + def test_query_that_returns_no_data(self): + fmt = """ + select * from {table} where + time > 0 and time < 10 and + geohash = 'hash1' and + user = 'user1' + """ + query = fmt.format(table=table_name) + ts_obj = self.client.ts_query('GeoCheckin', query) + self.assertEqual(len(ts_obj.columns), 0) + self.assertEqual(len(ts_obj.rows), 0) + + def test_query_that_matches_some_data(self): + fmt = """ + select * from {table} where + time > {t1} and time < {t2} and + geohash = 'hash1' and + user = 'user2' + """ + query = fmt.format( + table=table_name, + t1=self.tenMinsAgoMsec, + t2=self.nowMsec) + ts_obj = self.client.ts_query('GeoCheckin', query) + self.validate_data(ts_obj) + + def test_query_that_matches_more_data(self): + fmt = """ + select * from {table} where + time >= {t1} and time <= {t2} and + geohash = 'hash1' and + user = 'user2' + """ + query = fmt.format( + table=table_name, + t1=self.twentyMinsAgoMsec, + t2=self.nowMsec) + ts_obj = self.client.ts_query('GeoCheckin', query) + j = 0 + for i, want in enumerate(self.rows): + if want[2] == self.twentyFiveMinsAgo: + continue + got = ts_obj.rows[j] + j += 1 + self.assertListEqual(got, want) + + def test_get_with_invalid_key(self): + key = ['hash1', 'user2'] + with self.assertRaises(RiakError): + self.client.ts_get('GeoCheckin', key) + + def test_get_single_value(self): + key = ['hash1', 'user2', self.fiveMinsAgo] + ts_obj = self.client.ts_get('GeoCheckin', key) + self.assertIsNotNone(ts_obj) + self.validate_data(ts_obj) + + def test_get_single_value_via_table(self): + key = ['hash1', 'user2', self.fiveMinsAgo] + table = Table(self.client, 'GeoCheckin') + ts_obj = table.get(key) + self.assertIsNotNone(ts_obj) + self.validate_data(ts_obj) + + def test_stream_keys(self): + table = Table(self.client, 'GeoCheckin') + streamed_keys = [] + for keylist in table.stream_keys(): + self.assertNotEqual([], keylist) + streamed_keys += keylist + for key in keylist: + self.assertIsInstance(key, list) + self.assertEqual(len(key), 3) + self.assertEqual('hash1', key[0]) + self.assertEqual('user2', key[1]) + # TODO RTS-367 ENABLE + # self.assertIsInstance(key[2], datetime.datetime) + self.assertEqual(len(streamed_keys), 5) + + def test_delete_single_value(self): + key = ['hash1', 'user2', self.twentyFiveMinsAgo] + rslt = self.client.ts_delete('GeoCheckin', key) + self.assertTrue(rslt) + ts_obj = self.client.ts_get('GeoCheckin', key) + self.assertEqual(len(ts_obj.rows), 0) diff --git a/riak/tests/test_yokozuna.py b/riak/tests/test_yokozuna.py index 4310784a..52f9af88 100644 --- a/riak/tests/test_yokozuna.py +++ b/riak/tests/test_yokozuna.py @@ -1,24 +1,10 @@ # -*- coding: utf-8 -*- -""" -Copyright 2015 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - import platform -from . import RUN_YZ +from riak.tests import RUN_YZ +from riak.tests.base import IntegrationTestBase +from riak.tests.comparison import Comparison +from riak.tests.yz_setup import yzSetUp, yzTearDown + if platform.python_version() < '2.7': unittest = __import__('unittest2') else: @@ -37,11 +23,26 @@ def wait_for_yz_index(bucket, key, index=None): while len(bucket.search('_yz_rk:' + key, index=index)['docs']) == 0: pass +# YZ index on bucket of the same name +testrun_yz = {'btype': None, 'bucket': 'yzbucket', 'index': 'yzbucket'} +# YZ index on bucket of a different name +testrun_yz_index = {'btype': None, + 'bucket': 'yzindexbucket', + 'index': 'yzindex'} + + +def setUpModule(): + yzSetUp(testrun_yz, testrun_yz_index) + + +def tearDownModule(): + yzTearDown(testrun_yz, testrun_yz_index) + -class YZSearchTests(object): - @unittest.skipUnless(RUN_YZ, 'RUN_YZ is undefined') +@unittest.skipUnless(RUN_YZ, 'RUN_YZ is 0') +class YZSearchTests(IntegrationTestBase, unittest.TestCase, Comparison): def test_yz_search_from_bucket(self): - bucket = self.client.bucket(self.yz['bucket']) + bucket = self.client.bucket(testrun_yz['bucket']) bucket.new("user", {"user_s": "Z"}).store() wait_for_yz_index(bucket, "user") results = bucket.search("user_s:Z") @@ -51,63 +52,58 @@ def test_yz_search_from_bucket(self): self.assertIn('_yz_rk', result) self.assertEqual(u'user', result['_yz_rk']) self.assertIn('_yz_rb', result) - self.assertEqual(self.yz['bucket'], result['_yz_rb']) + self.assertEqual(testrun_yz['bucket'], result['_yz_rb']) self.assertIn('score', result) self.assertIn('user_s', result) self.assertEqual(u'Z', result['user_s']) - @unittest.skipUnless(RUN_YZ, 'RUN_YZ is undefined') def test_yz_search_index_using_bucket(self): - bucket = self.client.bucket(self.yz_index['bucket']) + bucket = self.client.bucket(testrun_yz_index['bucket']) bucket.new("feliz", {"name_s": "Felix", "species_s": "Felis catus"}).store() - wait_for_yz_index(bucket, "feliz", index=self.yz_index['index']) - results = bucket.search('name_s:Felix', index=self.yz_index['index']) + wait_for_yz_index(bucket, "feliz", index=testrun_yz_index['index']) + results = bucket.search('name_s:Felix', + index=testrun_yz_index['index']) self.assertEqual(1, len(results['docs'])) - @unittest.skipUnless(RUN_YZ, 'RUN_YZ is undefined') def test_yz_search_index_using_wrong_bucket(self): - bucket = self.client.bucket(self.yz_index['bucket']) + bucket = self.client.bucket(testrun_yz_index['bucket']) bucket.new("feliz", {"name_s": "Felix", "species_s": "Felis catus"}).store() - wait_for_yz_index(bucket, "feliz", index=self.yz_index['index']) + wait_for_yz_index(bucket, "feliz", index=testrun_yz_index['index']) with self.assertRaises(Exception): bucket.search('name_s:Felix') - @unittest.skipUnless(RUN_YZ, 'RUN_YZ is undefined') def test_yz_get_search_index(self): - index = self.client.get_search_index(self.yz['bucket']) - self.assertEqual(self.yz['bucket'], index['name']) + index = self.client.get_search_index(testrun_yz['bucket']) + self.assertEqual(testrun_yz['bucket'], index['name']) self.assertEqual('_yz_default', index['schema']) self.assertEqual(3, index['n_val']) with self.assertRaises(Exception): - self.client.get_search_index('NOT' + self.yz['bucket']) + self.client.get_search_index('NOT' + testrun_yz['bucket']) - @unittest.skipUnless(RUN_YZ, 'RUN_YZ is undefined') def test_yz_delete_search_index(self): # expected to fail, since there's an attached bucket with self.assertRaises(Exception): - self.client.delete_search_index(self.yz['bucket']) + self.client.delete_search_index(testrun_yz['bucket']) # detatch bucket from index then delete - b = self.client.bucket(self.yz['bucket']) + b = self.client.bucket(testrun_yz['bucket']) b.set_property('search_index', '_dont_index_') - self.assertTrue(self.client.delete_search_index(self.yz['bucket'])) + self.assertTrue(self.client.delete_search_index(testrun_yz['bucket'])) # create it again - self.client.create_search_index(self.yz['bucket'], '_yz_default', 3) - b = self.client.bucket(self.yz['bucket']) - b.set_property('search_index', self.yz['bucket']) + self.client.create_search_index(testrun_yz['bucket'], '_yz_default', 3) + b = self.client.bucket(testrun_yz['bucket']) + b.set_property('search_index', testrun_yz['bucket']) # Wait for index to apply indexes = [] - while self.yz['bucket'] not in indexes: + while testrun_yz['bucket'] not in indexes: indexes = [i['name'] for i in self.client.list_search_indexes()] - @unittest.skipUnless(RUN_YZ, 'RUN_YZ is undefined') def test_yz_list_search_indexes(self): indexes = self.client.list_search_indexes() - self.assertIn(self.yz['bucket'], [item['name'] for item in indexes]) + self.assertIn(testrun_yz['bucket'], [item['name'] for item in indexes]) self.assertLessEqual(1, len(indexes)) - @unittest.skipUnless(RUN_YZ, 'RUN_YZ is undefined') def test_yz_create_schema(self): content = """ @@ -143,7 +139,6 @@ def test_yz_create_schema(self): self.assertEqual(schema_name, schema['name']) self.assertEqual(content, schema['content']) - @unittest.skipUnless(RUN_YZ, 'RUN_YZ is undefined') def test_yz_create_bad_schema(self): bad_content = """ str: '%s'", cell) + ts_cell.varchar_value = str_to_bytes(cell) + elif (isinstance(cell, int) or + (PY2 and isinstance(cell, long))): # noqa + logging.debug("cell -> int/long: '%s'", cell) + ts_cell.sint64_value = cell + elif isinstance(cell, float): + ts_cell.double_value = cell + else: + t = type(cell) + raise RiakError("can't serialize type '{}', value '{}'" + .format(t, cell)) + + def _encode_timeseries_keyreq(self, table, key, req): + key_vals = None + if isinstance(key, list): + key_vals = key + else: + raise ValueError("key must be a list") + + req.table = str_to_bytes(table.name) + for cell in key_vals: + ts_cell = req.key.add() + self._encode_to_ts_cell(cell, ts_cell) + + def _encode_timeseries_listkeysreq(self, table, req, timeout=None): + req.table = str_to_bytes(table.name) + if timeout: + req.timeout = timeout + + def _encode_timeseries_put(self, tsobj, req): + """ + Fills an TsPutReq message with the appropriate data and + metadata from a TsObject. + + :param tsobj: a TsObject + :type tsobj: TsObject + :param req: the protobuf message to fill + :type req: riak_pb.TsPutReq + """ + req.table = str_to_bytes(tsobj.table.name) + + if tsobj.columns: + raise NotImplementedError("columns are not implemented yet") + + if tsobj.rows and isinstance(tsobj.rows, list): + for row in tsobj.rows: + tsr = req.rows.add() # NB: type riak_pb.TsRow + if not isinstance(row, list): + raise ValueError("TsObject row must be a list of values") + for cell in row: + tsc = tsr.cells.add() # NB: type riak_pb.TsCell + self._encode_to_ts_cell(cell, tsc) + else: + raise RiakError("TsObject requires a list of rows") + + def _decode_timeseries(self, resp, tsobj): + """ + Fills an TsObject with the appropriate data and + metadata from a TsQueryResp. + + :param resp: the protobuf message from which to process data + :type resp: riak_pb.TsQueryRsp or riak_pb.TsGetResp + :param tsobj: a TsObject + :type tsobj: TsObject + """ + if tsobj.columns is not None: + for col in resp.columns: + col_name = bytes_to_str(col.name) + col_type = col.type + col = (col_name, col_type) + tsobj.columns.append(col) + + for row in resp.rows: + tsobj.rows.append( + self._decode_timeseries_row(row, resp.columns)) + + def _decode_timeseries_row(self, tsrow, tscols=None): + """ + Decodes a TsRow into a list + + :param tsrow: the protobuf TsRow to decode. + :type tsrow: riak_pb.TsRow + :param tscols: the protobuf TsColumn data to help decode. + :type tscols: list + :rtype list + """ + row = [] + for i, cell in enumerate(tsrow.cells): + col = None + if tscols is not None: + col = tscols[i] + if cell.HasField('varchar_value'): + if col and col.type != riak_pb.TsColumnType.Value('VARCHAR'): + raise TypeError('expected VARCHAR column') + else: + row.append(bytes_to_str(cell.varchar_value)) + elif cell.HasField('sint64_value'): + if col and col.type != riak_pb.TsColumnType.Value('SINT64'): + raise TypeError('expected SINT64 column') + else: + row.append(cell.sint64_value) + elif cell.HasField('double_value'): + if col and col.type != riak_pb.TsColumnType.Value('DOUBLE'): + raise TypeError('expected DOUBLE column') + else: + row.append(cell.double_value) + elif cell.HasField('timestamp_value'): + if col and col.type != riak_pb.TsColumnType.Value('TIMESTAMP'): + raise TypeError('expected TIMESTAMP column') + else: + dt = self._datetime_from_unix_time_millis( + cell.timestamp_value) + row.append(dt) + elif cell.HasField('boolean_value'): + if col and col.type != riak_pb.TsColumnType.Value('BOOLEAN'): + raise TypeError('expected BOOLEAN column') + else: + row.append(cell.boolean_value) + else: + row.append(None) + return row + def _decode_preflist(self, item): """ Decodes a preflist response diff --git a/riak/transports/pbc/connection.py b/riak/transports/pbc/connection.py index 0bc58232..7f0e8b5d 100644 --- a/riak/transports/pbc/connection.py +++ b/riak/transports/pbc/connection.py @@ -175,7 +175,10 @@ def _recv_msg(self, expect=None): msg_code, = struct.unpack("B", self._inbuf[:1]) if msg_code is MSG_CODE_ERROR_RESP: err = self._parse_msg(msg_code, self._inbuf[1:]) - raise RiakError(bytes_to_str(err.errmsg)) + if err is None: + raise RiakError('no error provided!') + else: + raise RiakError(bytes_to_str(err.errmsg)) elif msg_code in MESSAGE_CLASSES: msg = self._parse_msg(msg_code, self._inbuf[1:]) else: diff --git a/riak/transports/pbc/stream.py b/riak/transports/pbc/stream.py index 88e7abac..6231d481 100644 --- a/riak/transports/pbc/stream.py +++ b/riak/transports/pbc/stream.py @@ -1,31 +1,14 @@ -""" -Copyright 2012 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - - import json from riak_pb.messages import ( MSG_CODE_LIST_KEYS_RESP, MSG_CODE_MAP_RED_RESP, MSG_CODE_LIST_BUCKETS_RESP, - MSG_CODE_INDEX_RESP + MSG_CODE_INDEX_RESP, + MSG_CODE_TS_LIST_KEYS_RESP, ) from riak.util import decode_index_value, bytes_to_str from riak.client.index_page import CONTINUATION +from riak.transports.pbc.codec import RiakPbcCodec from six import PY2 @@ -55,7 +38,7 @@ def next(self): self.finished = True raise - if(self._is_done(resp)): + if self._is_done(resp): self.finished = True return resp @@ -181,3 +164,27 @@ def next(self): def __next__(self): # Python 3.x Version return self.next() + + +class RiakPbcTsKeyStream(RiakPbcStream, RiakPbcCodec): + """ + Used internally by RiakPbcTransport to implement key-list streams. + """ + + _expect = MSG_CODE_TS_LIST_KEYS_RESP + + def next(self): + response = super(RiakPbcTsKeyStream, self).next() + + if response.done and len(response.keys) is 0: + raise StopIteration + + keys = [] + for tsrow in response.keys: + keys.append(self._decode_timeseries_row(tsrow)) + + return keys + + def __next__(self): + # Python 3.x Version + return self.next() diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index e385c698..517df987 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -1,34 +1,15 @@ -""" -Copyright 2015 Basho Technologies, Inc. -Copyright 2010 Rusty Klophaus -Copyright 2010 Justin Sheehy -Copyright 2009 Jay Baird - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - import riak_pb from riak import RiakError from riak.transports.transport import RiakTransport from riak.riak_object import VClock +from riak.ts_object import TsObject from riak.util import decode_index_value, str_to_bytes, bytes_to_str from riak.transports.pbc.connection import RiakPbcConnection from riak.transports.pbc.stream import (RiakPbcKeyStream, RiakPbcMapredStream, RiakPbcBucketStream, - RiakPbcIndexStream) + RiakPbcIndexStream, + RiakPbcTsKeyStream) from riak.transports.pbc.codec import RiakPbcCodec from six import PY2, PY3 @@ -79,7 +60,16 @@ MSG_CODE_DT_FETCH_REQ, MSG_CODE_DT_FETCH_RESP, MSG_CODE_DT_UPDATE_REQ, - MSG_CODE_DT_UPDATE_RESP + MSG_CODE_DT_UPDATE_RESP, + MSG_CODE_TS_PUT_REQ, + MSG_CODE_TS_PUT_RESP, + MSG_CODE_TS_QUERY_REQ, + MSG_CODE_TS_QUERY_RESP, + MSG_CODE_TS_LIST_KEYS_REQ, + MSG_CODE_TS_GET_REQ, + MSG_CODE_TS_GET_RESP, + MSG_CODE_TS_DEL_REQ, + MSG_CODE_TS_DEL_RESP ) @@ -233,6 +223,67 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, return robj + def ts_get(self, table, key): + req = riak_pb.TsGetReq() + self._encode_timeseries_keyreq(table, key, req) + + msg_code, ts_get_resp = self._request(MSG_CODE_TS_GET_REQ, req, + MSG_CODE_TS_GET_RESP) + + tsobj = TsObject(self._client, table, [], None) + self._decode_timeseries(ts_get_resp, tsobj) + return tsobj + + def ts_put(self, tsobj): + req = riak_pb.TsPutReq() + self._encode_timeseries_put(tsobj, req) + + msg_code, resp = self._request(MSG_CODE_TS_PUT_REQ, req, + MSG_CODE_TS_PUT_RESP) + + if resp is not None: + return True + else: + raise RiakError("missing response object") + + def ts_delete(self, table, key): + req = riak_pb.TsDelReq() + self._encode_timeseries_keyreq(table, key, req) + + msg_code, ts_del_resp = self._request(MSG_CODE_TS_DEL_REQ, req, + MSG_CODE_TS_DEL_RESP) + + if ts_del_resp is not None: + return True + else: + raise RiakError("missing response object") + + def ts_query(self, table, query, interpolations=None): + req = riak_pb.TsQueryReq() + req.query.base = str_to_bytes(query) + + msg_code, ts_query_resp = self._request(MSG_CODE_TS_QUERY_REQ, req, + MSG_CODE_TS_QUERY_RESP) + + tsobj = TsObject(self._client, table, [], []) + self._decode_timeseries(ts_query_resp, tsobj) + return tsobj + + def ts_stream_keys(self, table, timeout=None): + """ + Streams keys from a timeseries table, returning an iterator that + yields lists of keys. + """ + req = riak_pb.TsListKeysReq() + t = None + if self.client_timeouts() and timeout: + t = timeout + self._encode_timeseries_listkeysreq(table, req, t) + + self._send_msg(MSG_CODE_TS_LIST_KEYS_REQ, req) + + return RiakPbcTsKeyStream(self) + def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None, timeout=None): req = riak_pb.RpbDelReq() diff --git a/riak/transports/transport.py b/riak/transports/transport.py index a7428359..f9fcae6d 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -92,6 +92,24 @@ def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, """ raise NotImplementedError + def ts_put(self, tsobj): + """ + Stores a timeseries object. + """ + raise NotImplementedError + + def ts_query(self, table, query, interpolations=None): + """ + Query timeseries data. + """ + raise NotImplementedError + + def ts_stream_keys(self, table, timeout=None): + """ + Streams the list of keys for the table through an iterator. + """ + raise NotImplementedError + def get_buckets(self, bucket_type=None, timeout=None): """ Gets the list of buckets as strings. diff --git a/riak/ts_object.py b/riak/ts_object.py new file mode 100644 index 00000000..ef01baff --- /dev/null +++ b/riak/ts_object.py @@ -0,0 +1,43 @@ +from riak import RiakError +from riak.table import Table + + +class TsObject(object): + """ + The TsObject holds information about Timeseries data, plus the data + itself. + """ + def __init__(self, client, table, rows=[], columns=[]): + """ + Construct a new TsObject. + + :param client: A RiakClient object. + :type client: :class:`RiakClient ` + :param table: The table for the timeseries data as a Table object. + :type table: :class:`Table` + :param rows: An list of lists with timeseries data + :type rows: list + :param columns: An list of Column names and types. Optional. + :type columns: list + """ + + if not isinstance(table, Table): + raise ValueError('table must be an instance of Table.') + + self.client = client + self.table = table + + self.rows = rows + if not isinstance(self.rows, list): + raise RiakError("TsObject requires a list of rows") + + self.columns = columns + if self.columns is not None and not isinstance(self.columns, list): + raise RiakError("TsObject columns must be a list") + + def store(self): + """ + Store the timeseries data in Riak. + :rtype: boolean + """ + return self.client.ts_put(self) diff --git a/riak/util.py b/riak/util.py index 9a5b5a14..5dc3e61a 100644 --- a/riak/util.py +++ b/riak/util.py @@ -1,21 +1,3 @@ -""" -Copyright 2014 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - from __future__ import print_function import warnings from collections import Mapping diff --git a/setup.py b/setup.py index 549f2799..ba3474e1 100755 --- a/setup.py +++ b/setup.py @@ -1,23 +1,29 @@ #!/usr/bin/env python -import sys +import platform from setuptools import setup, find_packages from version import get_version from commands import preconfigure, configure, create_bucket_types, \ - setup_security, enable_security, disable_security + setup_security, enable_security, disable_security, setup_timeseries install_requires = ['six >= 1.8.0'] requires = ['six(>=1.8.0)'] -if sys.version_info < (2, 7, 9): +if platform.python_version() < '2.7.9': install_requires.append("pyOpenSSL >= 0.14") requires.append("pyOpenSSL(>=0.14)") -if sys.version_info < (3, ): + +if platform.python_version() < '3.0': + install_requires.append('protobuf >=2.4.1, <2.7.0') + requires.append('protobuf(>=2.4.1,<2.7.0)') install_requires.append("riak_pb >=2.0.0") requires.append("riak_pb(>=2.0.0)") else: + install_requires.append('python3_protobuf >=2.4.1, <2.6.0') + requires.append('python3_protobuf(>=2.4.1,<2.6.0)') install_requires.append("python3_riak_pb >=2.0.0") requires.append("python3_riak_pb(>=2.0.0)") + tests_require = [] -if sys.version_info < (2, 7): +if platform.python_version() < '2.7.0': tests_require.append("unittest2") setup( @@ -39,6 +45,7 @@ test_suite='riak.tests.suite', url='https://github.com/basho/riak-python-client', cmdclass={'create_bucket_types': create_bucket_types, + 'setup_timeseries': setup_timeseries, 'setup_security': setup_security, 'preconfigure': preconfigure, 'configure': configure, diff --git a/tox.ini b/tox.ini index 1bb27de4..87e74bb1 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,10 @@ # and then run "tox" from this directory. [tox] -envlist = py26, py279, py27, py33, py34 +envlist = py278, py27, py33, py34, py35 + +[testenv:py278] +basepython = {env:HOME}/.pyenv/versions/riak-py278/bin/python2.7 [testenv] install_command = pip install --upgrade {packages}