From d527055121a66220e5d2473ebfaadba2d937f0bf Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Sun, 18 Oct 2015 18:16:34 -0700 Subject: [PATCH 01/30] Implementing timeseries types and code --- .gitignore | 1 + docs/client.rst | 7 ++++ riak/__init__.py | 8 +++-- riak/bucket.py | 2 +- riak/client/operations.py | 18 ++++++++++ riak/table.py | 57 ++++++++++++++++++++++++++++++++ riak/tests/test_all.py | 3 ++ riak/transports/pbc/codec.py | 17 ++++++++++ riak/transports/pbc/transport.py | 17 +++++++++- riak/transports/transport.py | 6 ++++ riak/ts_object.py | 54 ++++++++++++++++++++++++++++++ 11 files changed, 185 insertions(+), 5 deletions(-) create mode 100644 riak/table.py create mode 100644 riak/ts_object.py diff --git a/.gitignore b/.gitignore index 34e7a5bb..553047bf 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ build/ dist/ riak.egg-info/ *.egg +.eggs/ #*# *~ diff --git a/docs/client.rst b/docs/client.rst index edf9d14a..a2a3b135 100644 --- a/docs/client.rst +++ b/docs/client.rst @@ -123,6 +123,13 @@ Key-level Operations .. automethod:: RiakClient.fetch_datatype .. automethod:: RiakClient.update_datatype +-------------------- +Timeseries Operations +-------------------- + +.. automethod:: RiakClient.ts_put +.. automethod:: RiakClient.ts_query + ---------------- Query Operations ---------------- diff --git a/riak/__init__.py b/riak/__init__.py index eddc69bc..3ba87f64 100644 --- a/riak/__init__.py +++ b/riak/__init__.py @@ -33,14 +33,16 @@ from riak.riak_error import RiakError, ConflictError from riak.client import RiakClient from riak.bucket import RiakBucket, BucketType +from riak.table import Table from riak.node import RiakNode from riak.riak_object import RiakObject from riak.mapreduce import RiakKeyFilter, RiakMapReduce, RiakLink -__all__ = ['RiakBucket', 'BucketType', 'RiakNode', 'RiakObject', 'RiakClient', - 'RiakMapReduce', 'RiakKeyFilter', 'RiakLink', 'RiakError', - 'ConflictError', 'ONE', 'ALL', 'QUORUM', 'key_filter'] +__all__ = ['RiakBucket', 'Table', 'BucketType', 'RiakNode', + 'RiakObject', 'RiakClient', 'RiakMapReduce', 'RiakKeyFilter', + 'RiakLink', 'RiakError', 'ConflictError', + 'ONE', 'ALL', 'QUORUM', 'key_filter'] ONE = "one" ALL = "all" diff --git a/riak/bucket.py b/riak/bucket.py index bb95726d..2df7faca 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -196,7 +196,7 @@ def new(self, key=None, data=None, content_type='application/json', def get(self, key, r=None, pr=None, timeout=None, include_context=None, basic_quorum=None, notfound_ok=None): """ - Retrieve an :class:`~riak.riak_object.RiakObject` or + Retrieve a :class:`~riak.riak_object.RiakObject` or :class:`~riak.datatypes.Datatype`, based on the presence and value of the :attr:`datatype ` bucket property. diff --git a/riak/client/operations.py b/riak/client/operations.py index ecda64af..41bee63a 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -553,6 +553,24 @@ def put(self, transport, robj, w=None, dw=None, pw=None, return_body=None, if_none_match=if_none_match, timeout=timeout) + @retryable + def ts_put(self, transport, tsobj, timeout=None): + """ + ts_put(tsobj, timeout=None) + + Stores time series data in the Riak cluster. + + .. note:: This request is automatically retried :attr:`retries` + times if it fails due to network error. + + :param tsobj: the time series object to store + :type tsobj: RiakTsObject + :param timeout: a timeout value in milliseconds + :type timeout: int + """ + _validate_timeout(timeout) + return transport.ts_put(tsobj, timeout=timeout) + @retryable def get(self, transport, robj, r=None, pr=None, timeout=None, basic_quorum=None, notfound_ok=None): diff --git a/riak/table.py b/riak/table.py new file mode 100644 index 00000000..5259b2cb --- /dev/null +++ b/riak/table.py @@ -0,0 +1,57 @@ +""" +Copyright 2015 Basho Technologies + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" +from six import string_types, PY2 + + +class RiakTable(object): + """ + The ``RiakTable`` object allows you to access properties on a Riak table + (bucket type) and query timeseries data. + """ + def __init__(self, client, name): + """ + Returns a new ``Table`` instance. + + :param client: A :class:`RiakClient ` + instance + :type client: :class:`RiakClient ` + :param name: The tables's name + :type name: string + """ + + if not isinstance(name, string_types): + raise TypeError('Bucket name must be a string') + + if PY2: + try: + name = name.encode('ascii') + except UnicodeError: + raise TypeError('Unicode table names are not supported.') + + self._client = client + self.name = name + + def query(self, key): + """ + Retrieve a bucket-type property. + + :param key: The property to retrieve. + :type key: string + :rtype: mixed + """ + return self.get_properties()[key] diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 992e4997..de4b6f62 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -238,6 +238,9 @@ def test_timeout_validation(self): with self.assertRaises(ValueError): self.client.put(obj, timeout=bad) + with self.assertRaises(ValueError): + self.client.ts_put(obj, timeout=bad) + with self.assertRaises(ValueError): self.client.get(obj, timeout=bad) diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index a3f26e8a..0b457376 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -622,3 +622,20 @@ def _encode_map_update(self, dtype, msg, op): msg.flag_op = riak_pb.MapUpdate.ENABLE else: msg.flag_op = riak_pb.MapUpdate.DISABLE + + def _encode_timeseries(self, tsobj, ts_put_req): + """ + Fills an TsPutReq message with the appropriate data and + metadata from a RiakTsObject. + + :param tsobj: a RiakTsObject + :type tsobj: RiakTsObject + :param ts_put_req: the protobuf message to fill + :type ts_put_req: riak_pb.TsPutReq + """ + ts_put_req.table = str_to_bytes(tsobj.table) + # TODO RTS-367 columns / rows + if tsobj.columns: + if tsobj.rows: + else: + raise RiakError("RiakTsObject requires rows") diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index c77dab2b..867a5e40 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -77,7 +77,9 @@ MSG_CODE_DT_FETCH_REQ, MSG_CODE_DT_FETCH_RESP, MSG_CODE_DT_UPDATE_REQ, - MSG_CODE_DT_UPDATE_RESP + MSG_CODE_DT_UPDATE_RESP, + MSG_CODE_TS_PUT_REQ, + MSG_CODE_TS_PUT_RESP ) @@ -231,6 +233,19 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, return robj + def ts_put(self, tsobj): + req = riak_pb.TsPutReq() + + self._encode_timeseries(tsobj, req) + + msg_code, resp = self._request(MSG_CODE_TS_PUT_REQ, req, + MSG_CODE_TS_PUT_RESP) + + if resp is not None: + return True + elif not robj.key: + raise RiakError("missing response object") + def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None, timeout=None): req = riak_pb.RpbDelReq() diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 85dcae43..e58f2ddf 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -85,6 +85,12 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=None, """ raise NotImplementedError + def ts_put(self, tsobj, timeout=None): + """ + Stores a time series object. + """ + raise NotImplementedError + def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None, timeout=None): """ diff --git a/riak/ts_object.py b/riak/ts_object.py new file mode 100644 index 00000000..2b855f2e --- /dev/null +++ b/riak/ts_object.py @@ -0,0 +1,54 @@ +""" +Copyright 2015 Basho Technologies + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" + +# TODO RTS-367 +# Should the table parameter be its own object that has a query method on it? +# Like Bucket? +class TsObject(object): + """ + The TsObject holds meta information about Timeseries data, + plus the data itself. + """ + def __init__(self, client, table, rows, columns=None): + """ + Construct a new TsObject. + + :param client: A RiakClient object. + :type client: :class:`RiakClient ` + :param table: The table for the timeseries data as a Table object. + :type table: :class:`Table` + :param rows: An array of arrays with timeseries data + :type rows: array + :param columns: An array Column names and types. Optional. + :type columns: array + """ + + if table is None or len(table) == 0: + raise ValueError('Table must either be a non-empty string.') + + self.client = client + self.table = table + # TODO RTS-367 rows, columns + + def store(self): + """ + Store the timeseries data in Riak. + :rtype: boolean + """ + + return self.client.ts_put(self) From a75843bab72331a997e3065bdfd309144ab784a0 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Mon, 19 Oct 2015 08:20:39 -0700 Subject: [PATCH 02/30] Table object, timeseries ops do not use timeout --- riak/__init__.py | 1 + riak/client/operations.py | 9 +++------ riak/table.py | 20 ++++++++++---------- riak/tests/test_all.py | 3 --- riak/transports/pbc/codec.py | 6 ++++++ riak/transports/transport.py | 16 +++++++++++----- 6 files changed, 31 insertions(+), 24 deletions(-) diff --git a/riak/__init__.py b/riak/__init__.py index 3ba87f64..415f6660 100644 --- a/riak/__init__.py +++ b/riak/__init__.py @@ -1,4 +1,5 @@ """ +Copyright 2015 Basho Technologies Copyright 2010 Rusty Klophaus Copyright 2010 Justin Sheehy Copyright 2009 Jay Baird diff --git a/riak/client/operations.py b/riak/client/operations.py index 41bee63a..063eca9e 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -554,9 +554,9 @@ def put(self, transport, robj, w=None, dw=None, pw=None, return_body=None, timeout=timeout) @retryable - def ts_put(self, transport, tsobj, timeout=None): + def ts_put(self, transport, tsobj): """ - ts_put(tsobj, timeout=None) + ts_put(tsobj) Stores time series data in the Riak cluster. @@ -565,11 +565,8 @@ def ts_put(self, transport, tsobj, timeout=None): :param tsobj: the time series object to store :type tsobj: RiakTsObject - :param timeout: a timeout value in milliseconds - :type timeout: int """ - _validate_timeout(timeout) - return transport.ts_put(tsobj, timeout=timeout) + return transport.ts_put(tsobj) @retryable def get(self, transport, robj, r=None, pr=None, timeout=None, diff --git a/riak/table.py b/riak/table.py index 5259b2cb..a739e9bb 100644 --- a/riak/table.py +++ b/riak/table.py @@ -18,9 +18,9 @@ from six import string_types, PY2 -class RiakTable(object): +class Table(object): """ - The ``RiakTable`` object allows you to access properties on a Riak table + The ``Table`` object allows you to access properties on a Riak table (bucket type) and query timeseries data. """ def __init__(self, client, name): @@ -30,12 +30,12 @@ def __init__(self, client, name): :param client: A :class:`RiakClient ` instance :type client: :class:`RiakClient ` - :param name: The tables's name + :param name: The table's name :type name: string """ if not isinstance(name, string_types): - raise TypeError('Bucket name must be a string') + raise TypeError('Table name must be a string') if PY2: try: @@ -46,12 +46,12 @@ def __init__(self, client, name): self._client = client self.name = name - def query(self, key): + def query(self, query, interpolations=None): """ - Retrieve a bucket-type property. + Queries a timeseries table. - :param key: The property to retrieve. - :type key: string - :rtype: mixed + :param query: The timeseries query. + :type query: string + :rtype: :class:`TsObject ` """ - return self.get_properties()[key] + return self.client.ts_query(query, interpolations) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index de4b6f62..992e4997 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -238,9 +238,6 @@ def test_timeout_validation(self): with self.assertRaises(ValueError): self.client.put(obj, timeout=bad) - with self.assertRaises(ValueError): - self.client.ts_put(obj, timeout=bad) - with self.assertRaises(ValueError): self.client.get(obj, timeout=bad) diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index 0b457376..e67a4780 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -639,3 +639,9 @@ def _encode_timeseries(self, tsobj, ts_put_req): if tsobj.rows: else: raise RiakError("RiakTsObject requires rows") + + def _decode_timeseries(self, ts_put_resp, tsobj): + """ + TODO RTS-367 + """ + raise NotImplementedError diff --git a/riak/transports/transport.py b/riak/transports/transport.py index e58f2ddf..4089bbda 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -85,16 +85,22 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=None, """ raise NotImplementedError - def ts_put(self, tsobj, timeout=None): + def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, + pw=None, timeout=None): """ - Stores a time series object. + Deletes an object. """ raise NotImplementedError - def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, - pw=None, timeout=None): + def ts_put(self, tsobj): """ - Deletes an object. + Stores a timeseries object. + """ + raise NotImplementedError + + def ts_query(self, query, interpolations=None): + """ + Query timeseries data. """ raise NotImplementedError From b40078b3bf65aa395c783e5f2cf432e7f7565cb2 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Thu, 22 Oct 2015 10:44:32 -0700 Subject: [PATCH 03/30] Add setup code for Timeseries, begin to add integration tests --- README.rst | 22 +++++++++ commands.py | 88 +++++++++++++++++++++-------------- riak/tests/__init__.py | 2 + riak/tests/test_all.py | 4 +- riak/tests/test_timeseries.py | 22 +++++++++ setup.py | 3 +- 6 files changed, 104 insertions(+), 37 deletions(-) create mode 100644 riak/tests/test_timeseries.py diff --git a/README.rst b/README.rst index 99bfe27b..85468584 100644 --- a/README.rst +++ b/README.rst @@ -179,6 +179,28 @@ You may alternately add these lines to `setup.cfg` To skip the bucket-type tests, set the ``SKIP_BTYPES`` environment variable to ``1``. +Testing Timeseries (Riak 2+) +------------------------------ + +To test timeseries data, you must run the ``setup_timeseries`` command, +which will create the bucket-types used in testing, or create them +manually yourself. It can be run like so (substituting ``$RIAK`` with +the root of your Riak install) + +.. code-block:: console + + ./setup.py setup_timeseries --riak-admin=$RIAK/bin/riak-admin + +You may alternately add these lines to `setup.cfg` + +.. code-block:: ini + + [setup_timeseries] + riak-admin=/Users/sean/dev/riak/rel/riak/bin/riak-admin + +To enable the timeseries tests, set the ``SKIP_TIMESERIES`` environment +variable to ``0``. + Testing Secondary Indexes ------------------------- diff --git a/commands.py b/commands.py index 06ee3039..0a736284 100644 --- a/commands.py +++ b/commands.py @@ -11,8 +11,10 @@ import os.path -__all__ = ['create_bucket_types', 'setup_security', 'enable_security', - 'disable_security', 'preconfigure', 'configure'] +__all__ = ['create_bucket_types', + 'setup_security', 'enable_security', 'disable_security', + 'setup_timeseries', + 'preconfigure', 'configure'] # Exception classes used by this module. @@ -72,36 +74,7 @@ def check_output(*popenargs, **kwargs): except ImportError: import json - -class create_bucket_types(Command): - """ - Creates bucket-types appropriate for testing. By default this will create: - - * `pytest-maps` with ``{"datatype":"map"}`` - * `pytest-sets` with ``{"datatype":"set"}`` - * `pytest-counters` with ``{"datatype":"counter"}`` - * `pytest-consistent` with ``{"consistent":true}`` - * `pytest-write-once` with ``{"write_once": true}`` - * `pytest-mr` - * `pytest` with ``{"allow_mult":false}`` - """ - - description = "create bucket-types used in integration tests" - - user_options = [ - ('riak-admin=', None, 'path to the riak-admin script') - ] - - _props = { - 'pytest-maps': {'datatype': 'map'}, - 'pytest-sets': {'datatype': 'set'}, - 'pytest-counters': {'datatype': 'counter'}, - 'pytest-consistent': {'consistent': True}, - 'pytest-write-once': {'write_once': True}, - 'pytest-mr': {}, - 'pytest': {'allow_mult': False} - } - +class bucket_type_commands: def initialize_options(self): self.riak_admin = None @@ -170,6 +143,53 @@ def _btype_command(self, *args): cmd.extend(args) return cmd +class create_bucket_types(bucket_type_commands, Command): + """ + Creates bucket-types appropriate for testing. By default this will create: + + * `pytest-maps` with ``{"datatype":"map"}`` + * `pytest-sets` with ``{"datatype":"set"}`` + * `pytest-counters` with ``{"datatype":"counter"}`` + * `pytest-consistent` with ``{"consistent":true}`` + * `pytest-write-once` with ``{"write_once": true}`` + * `pytest-mr` + * `pytest` with ``{"allow_mult":false}`` + """ + + description = "create bucket-types used in integration tests" + + user_options = [ + ('riak-admin=', None, 'path to the riak-admin script') + ] + + _props = { + 'pytest-maps': {'datatype': 'map'}, + 'pytest-sets': {'datatype': 'set'}, + 'pytest-counters': {'datatype': 'counter'}, + 'pytest-consistent': {'consistent': True}, + 'pytest-write-once': {'write_once': True}, + 'pytest-mr': {}, + 'pytest': {'allow_mult': False} + } + + +class setup_timeseries(bucket_type_commands, Command): + """ + Creates bucket-types appropriate for timeseries. By default this will create: + + * `GeoCheckin` with ``{"props": {"n_val": 3, "table_def": "CREATE TABLE GeoCheckin (geohash varchar not null, user varchar not null, time timestamp not null, weather varchar not null, temperature float, PRIMARY KEY((quantum(time, 15, m),user), time, user))"}}`` + """ + + description = "create bucket-types used in timeseries tests" + + user_options = [ + ('riak-admin=', None, 'path to the riak-admin script') + ] + + _props = { + 'GeoCheckin': {'n_val': 3, 'table_def': 'CREATE TABLE GeoCheckin (geohash varchar not null, user varchar not null, time timestamp not null, weather varchar not null, temperature float, PRIMARY KEY((quantum(time, 15, m),user), time, user))'}, + } + class security_commands(object): def check_security_command(self, *args): @@ -469,6 +489,4 @@ def run(self): for cmd_name in self.get_sub_commands(): self.run_command(cmd_name) - sub_commands = [('create_bucket_types', None), - ('setup_security', None) - ] + sub_commands = [('create_bucket_types', None), ('setup_security', None)] diff --git a/riak/tests/__init__.py b/riak/tests/__init__.py index d85447ff..46f092c9 100644 --- a/riak/tests/__init__.py +++ b/riak/tests/__init__.py @@ -36,6 +36,8 @@ SKIP_INDEXES = int(os.environ.get('SKIP_INDEXES', '1')) +SKIP_TIMESERIES = int(os.environ.get('SKIP_TIMESERIES', '1')) + SKIP_POOL = os.environ.get('SKIP_POOL') SKIP_RESOLVE = int(os.environ.get('SKIP_RESOLVE', '0')) SKIP_BTYPES = int(os.environ.get('SKIP_BTYPES', '0')) diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 2a6ef8cc..51e256f8 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -36,6 +36,7 @@ from riak.tests.test_btypes import BucketTypeTests from riak.tests.test_security import SecurityTests from riak.tests.test_datatypes import DatatypeIntegrationTests +from riak.tests.test_timeseries import TimeseriesTests from riak.tests import HOST, PB_HOST, PB_PORT, HTTP_HOST, HTTP_PORT, \ HAVE_PROTO, DUMMY_HTTP_PORT, DUMMY_PB_PORT, \ @@ -58,7 +59,6 @@ testrun_yz_index = {'btype': None, 'bucket': None, 'index': None} testrun_yz_mr = {'btype': None, 'bucket': None, 'index': None} - def setUpModule(): global testrun_search_bucket, testrun_props_bucket, \ testrun_sibs_bucket, testrun_yz, testrun_yz_index, testrun_yz_mr @@ -375,6 +375,7 @@ class RiakPbcTransportTestCase(BasicKVTests, BucketTypeTests, SecurityTests, DatatypeIntegrationTests, + TimeseriesTests, BaseTestCase, unittest.TestCase, test_six.Comparison): @@ -393,6 +394,7 @@ def test_uses_client_id_if_given(self): self.assertEqual(zero_client_id, c.client_id) +# NB: no Timeseries support in HTTP class RiakHttpTransportTestCase(BasicKVTests, KVFileTests, BucketPropsTest, diff --git a/riak/tests/test_timeseries.py b/riak/tests/test_timeseries.py new file mode 100644 index 00000000..d082d3df --- /dev/null +++ b/riak/tests/test_timeseries.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +import platform + +from . import SKIP_TIMESERIES + +if platform.python_version() < '2.7': + unittest = __import__('unittest2') +else: + import unittest + + +class TimeseriesTests(BaseTestCase, unittest.TestCase): + + @unittest.skipIf(SKIP_TIMESERIES == '1', "skip requested for timeseries tests") + def test_store(self): + table = self.client.table(self.table_name) + measurements = [ + [ ] + ] + obj = table.new(measurements) + result = obj.store() + self.assertTrue(result) diff --git a/setup.py b/setup.py index 549f2799..4f8acfe0 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup, find_packages from version import get_version from commands import preconfigure, configure, create_bucket_types, \ - setup_security, enable_security, disable_security + setup_security, enable_security, disable_security, setup_timeseries install_requires = ['six >= 1.8.0'] requires = ['six(>=1.8.0)'] @@ -39,6 +39,7 @@ test_suite='riak.tests.suite', url='https://github.com/basho/riak-python-client', cmdclass={'create_bucket_types': create_bucket_types, + 'setup_timeseries': setup_timeseries, 'setup_security': setup_security, 'preconfigure': preconfigure, 'configure': configure, From 9de23fdb032cf3afcc33bc9d5e51f43507cbe33c Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Sat, 24 Oct 2015 09:11:01 -0700 Subject: [PATCH 04/30] Refactor tests to allow running individuall. Still have to implement super().setUp() --- riak/tests/__init__.py | 5 +- riak/tests/base.py | 87 ++++++++++++ riak/tests/{test_six.py => comparison.py} | 18 +-- riak/tests/test_2i.py | 24 +--- riak/tests/test_all.py | 154 +--------------------- riak/tests/test_btypes.py | 25 +--- riak/tests/test_comparison.py | 23 +--- riak/tests/test_datatypes.py | 43 ++---- riak/tests/test_feature_detection.py | 19 +-- riak/tests/test_filters.py | 19 +-- riak/tests/test_kv.py | 30 +---- riak/tests/test_mapreduce.py | 56 ++++---- riak/tests/test_pool.py | 27 +--- riak/tests/test_search.py | 28 +--- riak/tests/test_security.py | 22 +--- riak/tests/test_timeseries.py | 24 +++- riak/tests/test_yokozuna.py | 88 +++++++------ riak/tests/yz_setup.py | 31 +++++ riak/transports/pbc/codec.py | 8 +- riak/transports/pbc/connection.py | 5 +- riak/transports/pbc/transport.py | 4 +- setup.py | 22 +++- 22 files changed, 289 insertions(+), 473 deletions(-) create mode 100644 riak/tests/base.py rename riak/tests/{test_six.py => comparison.py} (88%) create mode 100644 riak/tests/yz_setup.py diff --git a/riak/tests/__init__.py b/riak/tests/__init__.py index 46f092c9..85599d4b 100644 --- a/riak/tests/__init__.py +++ b/riak/tests/__init__.py @@ -19,6 +19,8 @@ HOST = os.environ.get('RIAK_TEST_HOST', '127.0.0.1') +PROTOCOL = 'pbc' + PB_HOST = os.environ.get('RIAK_TEST_PB_HOST', HOST) PB_PORT = int(os.environ.get('RIAK_TEST_PB_PORT', '8087')) @@ -30,7 +32,6 @@ DUMMY_HTTP_PORT = int(os.environ.get('DUMMY_HTTP_PORT', '1023')) DUMMY_PB_PORT = int(os.environ.get('DUMMY_PB_PORT', '1022')) - SKIP_SEARCH = int(os.environ.get('SKIP_SEARCH', '1')) RUN_YZ = int(os.environ.get('RUN_YZ', '0')) @@ -38,7 +39,7 @@ SKIP_TIMESERIES = int(os.environ.get('SKIP_TIMESERIES', '1')) -SKIP_POOL = os.environ.get('SKIP_POOL') +SKIP_POOL = int(os.environ.get('SKIP_POOL', '1')) SKIP_RESOLVE = int(os.environ.get('SKIP_RESOLVE', '0')) SKIP_BTYPES = int(os.environ.get('SKIP_BTYPES', '0')) diff --git a/riak/tests/base.py b/riak/tests/base.py new file mode 100644 index 00000000..dbd662e0 --- /dev/null +++ b/riak/tests/base.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +import random + +from riak.client import RiakClient +from riak.tests import HOST, PROTOCOL,PB_PORT, HTTP_PORT, SECURITY_CREDS + +testrun_search_bucket = 'searchbucket' +testrun_props_bucket = 'propsbucket' +testrun_sibs_bucket = 'sibsbucket' + +def setUpModule(): + + c = RiakClient(host=PB_HOST, http_port=HTTP_PORT, + pb_port=PB_PORT, credentials=SECURITY_CREDS) + + c.bucket(testrun_sibs_bucket).allow_mult = True + + if (not SKIP_SEARCH and not RUN_YZ): + b = c.bucket(testrun_search_bucket) + b.enable_search() + + +def tearDownModule(): + c = RiakClient(host=HTTP_HOST, http_port=HTTP_PORT, + pb_port=PB_PORT, credentials=SECURITY_CREDS) + + c.bucket(testrun_sibs_bucket).clear_properties() + c.bucket(testrun_props_bucket).clear_properties() + + if not SKIP_SEARCH and not RUN_YZ: + b = c.bucket(testrun_search_bucket) + b.clear_properties() + +class BaseTestCase(object): + + host = None + pb_port = None + http_port = None + credentials = None + + @staticmethod + def randint(): + return random.randint(1, 999999) + + @staticmethod + def randname(length=12): + out = '' + for i in range(length): + out += chr(random.randint(ord('a'), ord('z'))) + return out + + def create_client(self, host=None, http_port=None, pb_port=None, + protocol=None, credentials=None, + **client_args): + host = host or self.host or HOST + http_port = http_port or self.http_port or HTTP_PORT + pb_port = pb_port or self.pb_port or PB_PORT + + if protocol is None: + if hasattr(self, 'protocol') and (self.protocol is not None): + protocol = self.protocol + else: + protocol = PROTOCOL + + self.protocol = protocol + + credentials = credentials or SECURITY_CREDS + + return RiakClient(protocol=protocol, + host=host, + http_port=http_port, + credentials=credentials, + pb_port=pb_port, **client_args) + + def setUp(self): + self.table_name = 'GeoCheckin' + self.bucket_name = self.randname() + self.key_name = self.randname() + self.search_bucket = testrun_search_bucket + self.sibs_bucket = testrun_sibs_bucket + self.props_bucket = testrun_props_bucket + # self.yz = testrun_yz + # self.yz_index = testrun_yz_index + # self.yz_mr = testrun_yz_mr + self.credentials = SECURITY_CREDS + self.client = self.create_client() + diff --git a/riak/tests/test_six.py b/riak/tests/comparison.py similarity index 88% rename from riak/tests/test_six.py rename to riak/tests/comparison.py index c68dd150..30cde091 100644 --- a/riak/tests/test_six.py +++ b/riak/tests/comparison.py @@ -1,20 +1,4 @@ -""" -Copyright 2014 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" +# -*- coding: utf-8 -*- from six import PY2, PY3 import collections import warnings diff --git a/riak/tests/test_2i.py b/riak/tests/test_2i.py index 86d14999..68b2810a 100644 --- a/riak/tests/test_2i.py +++ b/riak/tests/test_2i.py @@ -1,32 +1,16 @@ # -*- coding: utf-8 -*- -""" -Copyright 2015 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - import platform from riak import RiakError -from . import SKIP_INDEXES +from riak.tests import SKIP_INDEXES +from riak.tests.base import BaseTestCase + if platform.python_version() < '2.7': unittest = __import__('unittest2') else: import unittest -class TwoITests(object): +class TwoITests(BaseTestCase, unittest.TestCase): def is_2i_supported(self): # Immediate test to see if 2i is even supported w/ the backend try: diff --git a/riak/tests/test_all.py b/riak/tests/test_all.py index 51e256f8..385fef7b 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_all.py @@ -1,22 +1,4 @@ # -*- coding: utf-8 -*- -""" -Copyright 2015 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" -import random import platform from six import PY2 from threading import Thread @@ -40,7 +22,7 @@ from riak.tests import HOST, PB_HOST, PB_PORT, HTTP_HOST, HTTP_PORT, \ HAVE_PROTO, DUMMY_HTTP_PORT, DUMMY_PB_PORT, \ - SKIP_SEARCH, RUN_YZ, SECURITY_CREDS, SKIP_POOL, test_six + SKIP_SEARCH, RUN_YZ, SECURITY_CREDS, SKIP_POOL if PY2: from Queue import Queue @@ -52,131 +34,6 @@ else: import unittest -testrun_search_bucket = None -testrun_props_bucket = None -testrun_sibs_bucket = None -testrun_yz = {'btype': None, 'bucket': None, 'index': None} -testrun_yz_index = {'btype': None, 'bucket': None, 'index': None} -testrun_yz_mr = {'btype': None, 'bucket': None, 'index': None} - -def setUpModule(): - global testrun_search_bucket, testrun_props_bucket, \ - testrun_sibs_bucket, testrun_yz, testrun_yz_index, testrun_yz_mr - - c = RiakClient(host=PB_HOST, http_port=HTTP_PORT, - pb_port=PB_PORT, credentials=SECURITY_CREDS) - - testrun_props_bucket = 'propsbucket' - testrun_sibs_bucket = 'sibsbucket' - c.bucket(testrun_sibs_bucket).allow_mult = True - - if (not SKIP_SEARCH and not RUN_YZ): - testrun_search_bucket = 'searchbucket' - b = c.bucket(testrun_search_bucket) - b.enable_search() - - if RUN_YZ: - # YZ index on bucket of the same name - testrun_yz = {'btype': None, 'bucket': 'yzbucket', - 'index': 'yzbucket'} - # YZ index on bucket of a different name - testrun_yz_index = {'btype': None, 'bucket': 'yzindexbucket', - 'index': 'yzindex'} - # Add bucket and type for Search 2.0 -> MapReduce - testrun_yz_mr = {'btype': 'pytest-mr', 'bucket': 'mrbucket', - 'index': 'mrbucket'} - - for yz in (testrun_yz, testrun_yz_index, testrun_yz_mr): - c.create_search_index(yz['index'], timeout=30000) - if yz['btype'] is not None: - t = c.bucket_type(yz['btype']) - b = t.bucket(yz['bucket']) - else: - b = c.bucket(yz['bucket']) - # Keep trying to set search bucket property until it succeeds - index_set = False - while not index_set: - try: - b.set_property('search_index', yz['index']) - index_set = True - except RiakError: - pass - - -def tearDownModule(): - global testrun_search_bucket, testrun_props_bucket, \ - testrun_sibs_bucket, testrun_yz_bucket - - c = RiakClient(host=HTTP_HOST, http_port=HTTP_PORT, - pb_port=PB_PORT, credentials=SECURITY_CREDS) - - c.bucket(testrun_sibs_bucket).clear_properties() - c.bucket(testrun_props_bucket).clear_properties() - - if not SKIP_SEARCH and not RUN_YZ: - b = c.bucket(testrun_search_bucket) - b.clear_properties() - - if RUN_YZ: - for yz in (testrun_yz, testrun_yz_index, testrun_yz_mr): - if yz['btype'] is not None: - t = c.bucket_type(yz['btype']) - b = t.bucket(yz['bucket']) - else: - b = c.bucket(yz['bucket']) - b.set_property('search_index', '_dont_index_') - c.delete_search_index(yz['index']) - for keys in b.stream_keys(): - for key in keys: - b.delete(key) - - -class BaseTestCase(object): - - host = None - pb_port = None - http_port = None - credentials = None - - @staticmethod - def randint(): - return random.randint(1, 999999) - - @staticmethod - def randname(length=12): - out = '' - for i in range(length): - out += chr(random.randint(ord('a'), ord('z'))) - return out - - def create_client(self, host=None, http_port=None, pb_port=None, - protocol=None, credentials=None, - **client_args): - host = host or self.host or HOST - http_port = http_port or self.http_port or HTTP_PORT - pb_port = pb_port or self.pb_port or PB_PORT - protocol = protocol or self.protocol - credentials = credentials or SECURITY_CREDS - return RiakClient(protocol=protocol, - host=host, - http_port=http_port, - credentials=credentials, - pb_port=pb_port, **client_args) - - def setUp(self): - self.bucket_name = self.randname() - self.key_name = self.randname() - self.search_bucket = testrun_search_bucket - self.sibs_bucket = testrun_sibs_bucket - self.props_bucket = testrun_props_bucket - self.yz = testrun_yz - self.yz_index = testrun_yz_index - self.yz_mr = testrun_yz_mr - self.credentials = SECURITY_CREDS - - self.client = self.create_client() - - class ClientTests(object): def test_request_retries(self): # We guess at some ports that will be unused by Riak or @@ -375,10 +232,7 @@ class RiakPbcTransportTestCase(BasicKVTests, BucketTypeTests, SecurityTests, DatatypeIntegrationTests, - TimeseriesTests, - BaseTestCase, - unittest.TestCase, - test_six.Comparison): + unittest.TestCase): def setUp(self): if not HAVE_PROTO: @@ -413,9 +267,7 @@ class RiakHttpTransportTestCase(BasicKVTests, BucketTypeTests, SecurityTests, DatatypeIntegrationTests, - BaseTestCase, - unittest.TestCase, - test_six.Comparison): + unittest.TestCase): def setUp(self): self.host = HTTP_HOST diff --git a/riak/tests/test_btypes.py b/riak/tests/test_btypes.py index b3ea5db2..e1992e04 100644 --- a/riak/tests/test_btypes.py +++ b/riak/tests/test_btypes.py @@ -1,25 +1,8 @@ -""" -Copyright 2015 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - import platform -from . import SKIP_BTYPES -from riak.bucket import RiakBucket, BucketType from riak import RiakError, RiakObject +from riak.bucket import RiakBucket, BucketType +from riak.tests import SKIP_BTYPES +from riak.tests.base import BaseTestCase if platform.python_version() < '2.7': unittest = __import__('unittest2') @@ -27,7 +10,7 @@ import unittest -class BucketTypeTests(object): +class BucketTypeTests(BaseTestCase, unittest.TestCase): def test_btype_init(self): btype = self.client.bucket_type('foo') self.assertIsInstance(btype, BucketType) diff --git a/riak/tests/test_comparison.py b/riak/tests/test_comparison.py index 38a1ef9f..c21bfee5 100644 --- a/riak/tests/test_comparison.py +++ b/riak/tests/test_comparison.py @@ -1,25 +1,8 @@ -""" -Copyright 2015 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - +# -*- coding: utf-8 -*- import platform from riak.riak_object import RiakObject from riak.bucket import RiakBucket, BucketType -from riak.tests.test_all import BaseTestCase +from riak.tests.base import BaseTestCase if platform.python_version() < '2.7': unittest = __import__('unittest2') @@ -153,7 +136,7 @@ def test_object_valid_key(self): self.assertIsNone(b, 'empty object key not allowed') -class RiakClientComparisonTest(unittest.TestCase, BaseTestCase): +class RiakClientComparisonTest(BaseTestCase, unittest.TestCase): def test_client_eq(self): self.protocol = 'http' a = self.create_client(host='host1', http_port=11) diff --git a/riak/tests/test_datatypes.py b/riak/tests/test_datatypes.py index 9c6b3a0b..5e5f0614 100644 --- a/riak/tests/test_datatypes.py +++ b/riak/tests/test_datatypes.py @@ -1,27 +1,10 @@ # -*- coding: utf-8 -*- -""" -Copyright 2015 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - import platform from riak import RiakBucket, BucketType, RiakObject import riak.datatypes as datatypes -from . import SKIP_DATATYPES -from riak.tests import test_six +from riak.tests import SKIP_DATATYPES +from riak.tests.base import BaseTestCase +from riak.tests.comparison import Comparison if platform.python_version() < '2.7': unittest = __import__('unittest2') @@ -29,7 +12,7 @@ import unittest -class DatatypeUnitTests(object): +class DatatypeUnitTestBase(object): dtype = None bucket = RiakBucket(None, 'test', BucketType(None, 'datatypes')) @@ -67,8 +50,7 @@ def test_op_output(self): self.check_op_output(op) -class FlagUnitTests(DatatypeUnitTests, - unittest.TestCase): +class FlagUnitTests(DatatypeUnitTestBase, unittest.TestCase): dtype = datatypes.Flag def op(self, dtype): @@ -87,8 +69,7 @@ def test_disables_require_context(self): self.assertTrue(dtype.modified) -class RegisterUnitTests(DatatypeUnitTests, - unittest.TestCase): +class RegisterUnitTests(DatatypeUnitTestBase, unittest.TestCase): dtype = datatypes.Register def op(self, dtype): @@ -98,8 +79,7 @@ def check_op_output(self, op): self.assertEqual(('assign', 'foobarbaz'), op) -class CounterUnitTests(DatatypeUnitTests, - unittest.TestCase): +class CounterUnitTests(DatatypeUnitTestBase, unittest.TestCase): dtype = datatypes.Counter def op(self, dtype): @@ -109,9 +89,7 @@ def check_op_output(self, op): self.assertEqual(('increment', 5), op) -class SetUnitTests(DatatypeUnitTests, - unittest.TestCase, - test_six.Comparison): +class SetUnitTests(DatatypeUnitTestBase, unittest.TestCase, Comparison): dtype = datatypes.Set def op(self, dtype): @@ -136,8 +114,7 @@ def test_removes_require_context(self): self.assertTrue(dtype.modified) -class MapUnitTests(DatatypeUnitTests, - unittest.TestCase): +class MapUnitTests(DatatypeUnitTestBase, unittest.TestCase): dtype = datatypes.Map def op(self, dtype): @@ -170,7 +147,7 @@ def test_removes_require_context(self): self.assertTrue(dtype.modified) -class DatatypeIntegrationTests(object): +class DatatypeIntegrationTests(BaseTestCase, unittest.TestCase): @unittest.skipIf(SKIP_DATATYPES, 'SKIP_DATATYPES is set') def test_dt_counter(self): btype = self.client.bucket_type('pytest-counters') diff --git a/riak/tests/test_feature_detection.py b/riak/tests/test_feature_detection.py index 682c5ac2..d88334aa 100644 --- a/riak/tests/test_feature_detection.py +++ b/riak/tests/test_feature_detection.py @@ -1,21 +1,4 @@ -""" -Copyright 2012-2015 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - +# -*- coding: utf-8 -*- import platform from riak.transports.feature_detect import FeatureDetection diff --git a/riak/tests/test_filters.py b/riak/tests/test_filters.py index 00e9d0af..c821ce95 100644 --- a/riak/tests/test_filters.py +++ b/riak/tests/test_filters.py @@ -1,21 +1,4 @@ -""" -Copyright 2015 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - +# -*- coding: utf-8 -*- import platform from riak.mapreduce import RiakKeyFilter from riak import key_filter diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index d1b28298..b1752e8f 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -1,22 +1,4 @@ # -*- coding: utf-8 -*- -""" -Copyright 2015 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - import os import platform from six import string_types, PY2, PY3 @@ -25,7 +7,9 @@ from time import sleep from riak import ConflictError, RiakBucket, RiakError from riak.resolver import default_resolver, last_written_resolver -from . import SKIP_RESOLVE +from riak.tests import SKIP_RESOLVE +from riak.tests.base import BaseTestCase +from riak.tests.comparison import Comparison try: import simplejson as json @@ -71,7 +55,7 @@ def __eq__(self, other): return True -class BasicKVTests(object): +class BasicKVTests(BaseTestCase, unittest.TestCase, Comparison): def test_is_alive(self): self.assertTrue(self.client.is_alive()) @@ -610,7 +594,7 @@ def generate_siblings(self, original, count=5, delay=None): return vals -class BucketPropsTest(object): +class BucketPropsTest(BaseTestCase, unittest.TestCase): def test_rw_settings(self): bucket = self.client.bucket(self.props_bucket) self.assertEqual(bucket.r, "quorum") @@ -663,7 +647,7 @@ def test_clear_bucket_properties(self): self.assertEqual(bucket.n_val, 3) -class KVFileTests(object): +class KVFileTests(BaseTestCase, unittest.TestCase): def test_store_binary_object_from_file(self): bucket = self.client.bucket(self.bucket_name) filepath = os.path.join(os.path.dirname(__file__), 'test_all.py') @@ -691,7 +675,7 @@ def test_store_binary_object_from_file_should_fail_if_file_not_found(self): self.assertFalse(obj.exists) -class CounterTests(object): +class CounterTests(BaseTestCase, unittest.TestCase): def test_counter_requires_allow_mult(self): bucket = self.client.bucket(self.bucket_name) if bucket.allow_mult: diff --git a/riak/tests/test_mapreduce.py b/riak/tests/test_mapreduce.py index c15ff7b1..99ed8a92 100644 --- a/riak/tests/test_mapreduce.py +++ b/riak/tests/test_mapreduce.py @@ -1,38 +1,38 @@ # -*- coding: utf-8 -*- -""" -Copyright 2015 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 +from __future__ import print_function -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" +import platform -from __future__ import print_function from six import PY2 from riak.mapreduce import RiakMapReduce -from riak import key_filter, RiakError +from riak import key_filter, RiakClient, RiakError +from riak.tests import RUN_YZ, PB_HOST, PB_PORT, HTTP_HOST, HTTP_PORT, SECURITY_CREDS +from riak.tests.base import BaseTestCase from riak.tests.test_yokozuna import wait_for_yz_index from riak.tests import RUN_SECURITY -import platform +from riak.tests.yz_setup import yzSetUpModule, yzTearDownModule -from . import RUN_YZ if platform.python_version() < '2.7': unittest = __import__('unittest2') else: import unittest +# Add bucket and type for Search 2.0 -> MapReduce +testrun_yz_mr = {'btype': 'pytest-mr', 'bucket': 'mrbucket', 'index': 'mrbucket'} + +def setUpModule(): + if RUN_YZ: + c = RiakClient(host=PB_HOST, http_port=HTTP_PORT, + pb_port=PB_PORT, credentials=SECURITY_CREDS) + yzSetUpModule(c, testrun_yz_mr) + +def tearDownModule(): + if RUN_YZ: + c = RiakClient(host=HTTP_HOST, http_port=HTTP_PORT, + pb_port=PB_PORT, credentials=SECURITY_CREDS) + yzTearDownModule(c, testrun_yz_mr) -class LinkTests(object): +class LinkTests(BaseTestCase, unittest.TestCase): def test_store_and_get_links(self): # Create the object... bucket = self.client.bucket(self.bucket_name) @@ -98,7 +98,7 @@ def test_link_walking(self): self.assertEqual(len(results), 1) -class ErlangMapReduceTests(object): +class ErlangMapReduceTests(BaseTestCase, unittest.TestCase): def test_erlang_map_reduce(self): # Create the object... bucket = self.client.bucket(self.bucket_name) @@ -204,7 +204,7 @@ def test_client_exceptional_paths(self): mr.add_key_filter("tokenize", "-", 1) -class JSMapReduceTests(object): +class JSMapReduceTests(BaseTestCase, unittest.TestCase): def test_javascript_source_map(self): # Create the object... bucket = self.client.bucket(self.bucket_name) @@ -525,8 +525,8 @@ def test_mr_search(self): """ Try a successful map/reduce from search results. """ - btype = self.client.bucket_type(self.yz_mr['btype']) - bucket = btype.bucket(self.yz_mr['bucket']) + btype = self.client.bucket_type(testrun_yz_mr['btype']) + bucket = btype.bucket(testrun_yz_mr['bucket']) bucket.new("Pebbles", {"name_s": "Fruity Pebbles", "maker_s": "Post", "sugar_i": 9, @@ -554,7 +554,7 @@ def test_mr_search(self): "fruit_b": False}).store() # Wait for Solr to catch up wait_for_yz_index(bucket, "Crunch") - mr = RiakMapReduce(self.client).search(self.yz_mr['bucket'], + mr = RiakMapReduce(self.client).search(testrun_yz_mr['bucket'], 'fruit_b:false') mr.map("""function(v) { var solr_doc = JSON.parse(v.values[0].data); @@ -564,7 +564,7 @@ def test_mr_search(self): self.assertEqual(result, [100]) -class MapReduceAliasTests(object): +class MapReduceAliasTests(BaseTestCase, unittest.TestCase): """This tests the map reduce aliases""" def test_map_values(self): @@ -759,7 +759,7 @@ def test_filter_not_found(self): self.assertEqual(sorted(result), [1, 2]) -class MapReduceStreamTests(object): +class MapReduceStreamTests(BaseTestCase, unittest.TestCase): def test_stream_results(self): bucket = self.client.bucket(self.bucket_name) bucket.new('one', data=1).store() diff --git a/riak/tests/test_pool.py b/riak/tests/test_pool.py index 6355eee0..1f0e2f19 100644 --- a/riak/tests/test_pool.py +++ b/riak/tests/test_pool.py @@ -1,21 +1,4 @@ -""" -Copyright 2015 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - +# -*- coding: utf-8 -*- from six import PY2 import platform from threading import Thread, currentThread @@ -23,7 +6,7 @@ from random import SystemRandom from time import sleep from . import SKIP_POOL -from riak.tests import test_six +from riak.tests.comparison import Comparison if platform.python_version() < '2.7': unittest = __import__('unittest2') @@ -54,10 +37,8 @@ def create_resource(self): return [] -@unittest.skipIf(SKIP_POOL, - 'Skipping connection pool tests') -class PoolTest(unittest.TestCase, - test_six.Comparison): +@unittest.skipIf(SKIP_POOL, 'Skipping connection pool tests') +class PoolTest(unittest.TestCase, Comparison): def test_yields_new_object_when_empty(self): """ diff --git a/riak/tests/test_search.py b/riak/tests/test_search.py index eed22e2c..11f9f4c9 100644 --- a/riak/tests/test_search.py +++ b/riak/tests/test_search.py @@ -1,32 +1,16 @@ # -*- coding: utf-8 -*- -""" -Copyright 2015 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - from __future__ import print_function import platform -from . import SKIP_SEARCH +from riak.tests import SKIP_SEARCH +from riak.tests.base import BaseTestCase + if platform.python_version() < '2.7': unittest = __import__('unittest2') else: import unittest -class EnableSearchTests(object): +class EnableSearchTests(BaseTestCase, unittest.TestCase): @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_bucket_search_enabled(self): bucket = self.client.bucket(self.bucket_name) @@ -57,7 +41,7 @@ def test_disable_search_commit_hook(self): bucket.enable_search() -class SolrSearchTests(object): +class SolrSearchTests(BaseTestCase, unittest.TestCase): @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_add_document_to_index(self): self.client.fulltext_add(self.search_bucket, @@ -116,7 +100,7 @@ def test_delete_documents_from_search_by_query_and_id(self): self.assertEqual(0, len(results['docs'])) -class SearchTests(object): +class SearchTests(BaseTestCase, unittest.TestCase): @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_solr_search_from_bucket(self): bucket = self.client.bucket(self.search_bucket) diff --git a/riak/tests/test_security.py b/riak/tests/test_security.py index f0489039..065e16ee 100644 --- a/riak/tests/test_security.py +++ b/riak/tests/test_security.py @@ -1,35 +1,19 @@ # -*- coding: utf-8 -*- -""" -Copyright 2015 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - import sys from riak.tests import RUN_SECURITY, SECURITY_USER, SECURITY_PASSWD, \ SECURITY_CACERT, SECURITY_KEY, SECURITY_CERT, SECURITY_REVOKED, \ SECURITY_CERT_USER, SECURITY_CERT_PASSWD, SECURITY_BAD_CERT, \ SECURITY_CREDS, SECURITY_CIPHERS from riak.security import SecurityCreds +from riak.tests.base import BaseTestCase + if sys.version_info < (2, 7): unittest = __import__('unittest2') else: import unittest -class SecurityTests(object): +class SecurityTests(BaseTestCase, unittest.TestCase): @unittest.skipIf(RUN_SECURITY, 'RUN_SECURITY is set') def test_security_disabled(self): client = self.create_client(credentials=SECURITY_CREDS) diff --git a/riak/tests/test_timeseries.py b/riak/tests/test_timeseries.py index d082d3df..2bfa4585 100644 --- a/riak/tests/test_timeseries.py +++ b/riak/tests/test_timeseries.py @@ -1,22 +1,36 @@ # -*- coding: utf-8 -*- import platform +import time +import sys -from . import SKIP_TIMESERIES +from riak.tests import SKIP_TIMESERIES +from riak.tests.base import BaseTestCase if platform.python_version() < '2.7': unittest = __import__('unittest2') else: import unittest - class TimeseriesTests(BaseTestCase, unittest.TestCase): + def setUp(self): + super(TimeseriesTests, self).setUp() @unittest.skipIf(SKIP_TIMESERIES == '1', "skip requested for timeseries tests") def test_store(self): + now = int(round(time.time() * 1000)) # NB: millis since Jan 1 1970 UTC + fiveMinsInMsec = 5 * 60 * 1000 + fiveMinsAgo = now - fiveMinsInMsec + tenMinsAgo = fiveMinsAgo - fiveMinsInMsec + fifteenMinsAgo = tenMinsAgo - fiveMinsInMsec + twentyMinsAgo = fifteenMinsAgo - fiveMinsInMsec + table = self.client.table(self.table_name) measurements = [ - [ ] + [ 'hash1', 'user2', twentyMinsAgo, 'hurricane', '84.3' ], + [ 'hash1', 'user2', fifteenMinsAgo, 'rain', '79.0' ], + [ 'hash1', 'user2', fiveMinsAgo, 'wind', 50.5 ], + [ 'hash1', 'user2', now, 'snow', 20.1 ] ] - obj = table.new(measurements) - result = obj.store() + ts_obj = table.new(measurements) + result = ts_obj.store() self.assertTrue(result) diff --git a/riak/tests/test_yokozuna.py b/riak/tests/test_yokozuna.py index 4310784a..521be2e4 100644 --- a/riak/tests/test_yokozuna.py +++ b/riak/tests/test_yokozuna.py @@ -1,29 +1,32 @@ # -*- coding: utf-8 -*- -""" -Copyright 2015 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - import platform -from . import RUN_YZ +from riak import RiakClient +from riak.tests import RUN_YZ, PB_HOST, PB_PORT, HTTP_PORT, SECURITY_CREDS +from riak.tests.base import BaseTestCase +from riak.tests.yz_setup import yzSetUpModule, yzTearDownModule + if platform.python_version() < '2.7': unittest = __import__('unittest2') else: import unittest +# YZ index on bucket of the same name +testrun_yz = {'btype': None, 'bucket': 'yzbucket', 'index': 'yzbucket'} +# YZ index on bucket of a different name +testrun_yz_index = {'btype': None, 'bucket': 'yzindexbucket', 'index': 'yzindex'} + +def setUpModule(): + if RUN_YZ: + c = RiakClient(host=PB_HOST, protocol='pbc', + pb_port=PB_PORT, credentials=SECURITY_CREDS) + yzSetUpModule(c, testrun_yz, testrun_yz_index) + +def tearDownModule(): + if RUN_YZ: + c = RiakClient(host=PB_HOST, protocol='pbc', + pb_port=PB_PORT, credentials=SECURITY_CREDS) + yzTearDownModule(c, testrun_yz, testrun_yz_index) + def wait_for_yz_index(bucket, key, index=None): """ @@ -38,10 +41,13 @@ def wait_for_yz_index(bucket, key, index=None): pass -class YZSearchTests(object): +class YZSearchTests(BaseTestCase, unittest.TestCase): + def setUp(self): + super(YZSearchTests, self).setUp() + @unittest.skipUnless(RUN_YZ, 'RUN_YZ is undefined') def test_yz_search_from_bucket(self): - bucket = self.client.bucket(self.yz['bucket']) + bucket = self.client.bucket(testrun_yz['bucket']) bucket.new("user", {"user_s": "Z"}).store() wait_for_yz_index(bucket, "user") results = bucket.search("user_s:Z") @@ -51,60 +57,60 @@ def test_yz_search_from_bucket(self): self.assertIn('_yz_rk', result) self.assertEqual(u'user', result['_yz_rk']) self.assertIn('_yz_rb', result) - self.assertEqual(self.yz['bucket'], result['_yz_rb']) + self.assertEqual(testrun_yz['bucket'], result['_yz_rb']) self.assertIn('score', result) self.assertIn('user_s', result) self.assertEqual(u'Z', result['user_s']) @unittest.skipUnless(RUN_YZ, 'RUN_YZ is undefined') def test_yz_search_index_using_bucket(self): - bucket = self.client.bucket(self.yz_index['bucket']) + bucket = self.client.bucket(testrun_yz_index['bucket']) bucket.new("feliz", {"name_s": "Felix", "species_s": "Felis catus"}).store() - wait_for_yz_index(bucket, "feliz", index=self.yz_index['index']) - results = bucket.search('name_s:Felix', index=self.yz_index['index']) + wait_for_yz_index(bucket, "feliz", index=testrun_yz_index['index']) + results = bucket.search('name_s:Felix', index=testrun_yz_index['index']) self.assertEqual(1, len(results['docs'])) @unittest.skipUnless(RUN_YZ, 'RUN_YZ is undefined') def test_yz_search_index_using_wrong_bucket(self): - bucket = self.client.bucket(self.yz_index['bucket']) + bucket = self.client.bucket(testrun_yz_index['bucket']) bucket.new("feliz", {"name_s": "Felix", "species_s": "Felis catus"}).store() - wait_for_yz_index(bucket, "feliz", index=self.yz_index['index']) + wait_for_yz_index(bucket, "feliz", index=testrun_yz_index['index']) with self.assertRaises(Exception): bucket.search('name_s:Felix') @unittest.skipUnless(RUN_YZ, 'RUN_YZ is undefined') def test_yz_get_search_index(self): - index = self.client.get_search_index(self.yz['bucket']) - self.assertEqual(self.yz['bucket'], index['name']) + index = self.client.get_search_index(testrun_yz['bucket']) + self.assertEqual(testrun_yz['bucket'], index['name']) self.assertEqual('_yz_default', index['schema']) self.assertEqual(3, index['n_val']) with self.assertRaises(Exception): - self.client.get_search_index('NOT' + self.yz['bucket']) + self.client.get_search_index('NOT' + testrun_yz['bucket']) @unittest.skipUnless(RUN_YZ, 'RUN_YZ is undefined') def test_yz_delete_search_index(self): # expected to fail, since there's an attached bucket with self.assertRaises(Exception): - self.client.delete_search_index(self.yz['bucket']) + self.client.delete_search_index(testrun_yz['bucket']) # detatch bucket from index then delete - b = self.client.bucket(self.yz['bucket']) + b = self.client.bucket(testrun_yz['bucket']) b.set_property('search_index', '_dont_index_') - self.assertTrue(self.client.delete_search_index(self.yz['bucket'])) + self.assertTrue(self.client.delete_search_index(testrun_yz['bucket'])) # create it again - self.client.create_search_index(self.yz['bucket'], '_yz_default', 3) - b = self.client.bucket(self.yz['bucket']) - b.set_property('search_index', self.yz['bucket']) + self.client.create_search_index(testrun_yz['bucket'], '_yz_default', 3) + b = self.client.bucket(testrun_yz['bucket']) + b.set_property('search_index', testrun_yz['bucket']) # Wait for index to apply indexes = [] - while self.yz['bucket'] not in indexes: + while testrun_yz['bucket'] not in indexes: indexes = [i['name'] for i in self.client.list_search_indexes()] @unittest.skipUnless(RUN_YZ, 'RUN_YZ is undefined') def test_yz_list_search_indexes(self): indexes = self.client.list_search_indexes() - self.assertIn(self.yz['bucket'], [item['name'] for item in indexes]) + self.assertIn(testrun_yz['bucket'], [item['name'] for item in indexes]) self.assertLessEqual(1, len(indexes)) @unittest.skipUnless(RUN_YZ, 'RUN_YZ is undefined') @@ -153,7 +159,7 @@ def test_yz_create_bad_schema(self): @unittest.skipUnless(RUN_YZ, 'RUN_YZ is undefined') def test_yz_search_queries(self): - bucket = self.client.bucket(self.yz['bucket']) + bucket = self.client.bucket(testrun_yz['bucket']) bucket.new("Z", {"username_s": "Z", "name_s": "ryan", "age_i": 30}).store() bucket.new("R", {"username_s": "R", "name_s": "eric", @@ -191,7 +197,7 @@ def test_yz_search_queries(self): @unittest.skipUnless(RUN_YZ, 'RUN_YZ is undefined') def test_yz_search_utf8(self): - bucket = self.client.bucket(self.yz['bucket']) + bucket = self.client.bucket(testrun_yz['bucket']) body = {"text_ja": u"私はハイビスカスを食べるのが 大好き"} bucket.new(self.key_name, body).store() wait_for_yz_index(bucket, self.key_name) @@ -201,7 +207,7 @@ def test_yz_search_utf8(self): @unittest.skipUnless(RUN_YZ, 'RUN_YZ is undefined') def test_yz_multivalued_fields(self): - bucket = self.client.bucket(self.yz['bucket']) + bucket = self.client.bucket(testrun_yz['bucket']) body = {"groups_ss": ['a', 'b', 'c']} bucket.new(self.key_name, body).store() wait_for_yz_index(bucket, self.key_name) diff --git a/riak/tests/yz_setup.py b/riak/tests/yz_setup.py new file mode 100644 index 00000000..d0e83147 --- /dev/null +++ b/riak/tests/yz_setup.py @@ -0,0 +1,31 @@ +from riak import RiakError + +def yzSetUpModule(c, *yzdata): + for yz in yzdata: + c.create_search_index(yz['index'], timeout=30000) + if yz['btype'] is not None: + t = c.bucket_type(yz['btype']) + b = t.bucket(yz['bucket']) + else: + b = c.bucket(yz['bucket']) + # Keep trying to set search bucket property until it succeeds + index_set = False + while not index_set: + try: + b.set_property('search_index', yz['index']) + index_set = True + except RiakError: + pass + +def yzTearDownModule(c, *yzdata): + for yz in yzdata: + if yz['btype'] is not None: + t = c.bucket_type(yz['btype']) + b = t.bucket(yz['bucket']) + else: + b = c.bucket(yz['bucket']) + b.set_property('search_index', '_dont_index_') + c.delete_search_index(yz['index']) + for keys in b.stream_keys(): + for key in keys: + b.delete(key) diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index 19eabf75..e8b47420 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -636,10 +636,10 @@ def _encode_timeseries(self, tsobj, ts_put_req): """ ts_put_req.table = str_to_bytes(tsobj.table) # TODO RTS-367 columns / rows - if tsobj.columns: - if tsobj.rows: - else: - raise RiakError("RiakTsObject requires rows") + # if tsobj.columns: + # if tsobj.rows: + # else: + # raise RiakError("RiakTsObject requires rows") def _decode_timeseries(self, ts_put_resp, tsobj): """ diff --git a/riak/transports/pbc/connection.py b/riak/transports/pbc/connection.py index 0bc58232..7f0e8b5d 100644 --- a/riak/transports/pbc/connection.py +++ b/riak/transports/pbc/connection.py @@ -175,7 +175,10 @@ def _recv_msg(self, expect=None): msg_code, = struct.unpack("B", self._inbuf[:1]) if msg_code is MSG_CODE_ERROR_RESP: err = self._parse_msg(msg_code, self._inbuf[1:]) - raise RiakError(bytes_to_str(err.errmsg)) + if err is None: + raise RiakError('no error provided!') + else: + raise RiakError(bytes_to_str(err.errmsg)) elif msg_code in MESSAGE_CLASSES: msg = self._parse_msg(msg_code, self._inbuf[1:]) else: diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 15b69f6f..5e3e91c8 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -80,8 +80,8 @@ MSG_CODE_DT_FETCH_RESP, MSG_CODE_DT_UPDATE_REQ, MSG_CODE_DT_UPDATE_RESP, - MSG_CODE_TS_PUT_REQ, - MSG_CODE_TS_PUT_RESP + # MSG_CODE_TS_PUT_REQ, + # MSG_CODE_TS_PUT_RESP ) diff --git a/setup.py b/setup.py index 4f8acfe0..dfcce1f3 100755 --- a/setup.py +++ b/setup.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +import os import sys from setuptools import setup, find_packages from version import get_version @@ -10,12 +11,23 @@ if sys.version_info < (2, 7, 9): install_requires.append("pyOpenSSL >= 0.14") requires.append("pyOpenSSL(>=0.14)") -if sys.version_info < (3, ): - install_requires.append("riak_pb >=2.0.0") - requires.append("riak_pb(>=2.0.0)") + +riak_pb_in_pythonpath = False +PYTHONPATH = os.environ.get('PYTHONPATH') +if PYTHONPATH is not None and PYTHONPATH.find('riak_pb/python/lib') != -1: + riak_pb_in_pythonpath = True + +if riak_pb_in_pythonpath: + install_requires.append("protobuf ==2.6.1") + requires.append("protobuf(==2.6.1)") else: - install_requires.append("python3_riak_pb >=2.0.0") - requires.append("python3_riak_pb(>=2.0.0)") + if sys.version_info < (3, ): + install_requires.append("riak_pb >=2.0.0") + requires.append("riak_pb(>=2.0.0)") + else: + install_requires.append("python3_riak_pb >=2.0.0") + requires.append("python3_riak_pb(>=2.0.0)") + tests_require = [] if sys.version_info < (2, 7): tests_require.append("unittest2") From a8227e1d837a938b952a7f4297d948279be1f9dd Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Sun, 25 Oct 2015 11:36:37 -0700 Subject: [PATCH 05/30] Refactored unit tests to allow running explicit test suites. Moved setUp / tearDown into tests that use them. --- .gitignore | 2 + buildbot/Makefile | 6 +- riak/tests/__init__.py | 2 +- riak/tests/base.py | 56 ++++------ riak/tests/test_2i.py | 28 ++--- riak/tests/test_btypes.py | 5 +- riak/tests/{test_all.py => test_client.py} | 114 ++------------------- riak/tests/test_comparison.py | 4 +- riak/tests/test_datatypes.py | 4 +- riak/tests/test_kv.py | 73 +++++++++---- riak/tests/test_mapreduce.py | 32 +++--- riak/tests/test_pool.py | 2 +- riak/tests/test_search.py | 82 +++++++++------ riak/tests/test_security.py | 15 ++- riak/tests/test_timeseries.py | 41 ++++---- riak/tests/test_yokozuna.py | 39 +++---- riak/tests/yz_setup.py | 69 +++++++------ riak/transports/http/__init__.py | 17 ++- riak/transports/http/connection.py | 19 ++-- riak/transports/http/transport.py | 3 +- setup.py | 11 +- 21 files changed, 299 insertions(+), 325 deletions(-) rename riak/tests/{test_all.py => test_client.py} (63%) diff --git a/.gitignore b/.gitignore index f9515221..24c0bded 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ *.pyc .python-version +.tox/ + docs/_build .*.swp diff --git a/buildbot/Makefile b/buildbot/Makefile index 1323a276..1209ac84 100644 --- a/buildbot/Makefile +++ b/buildbot/Makefile @@ -26,12 +26,14 @@ test: setup test_normal test_security test_normal: @echo "Testing Riak Python Client (without security)" @../setup.py disable_security --riak-admin=${RIAK_ADMIN} - @RUN_YZ=1 SKIP_DATATYPES=0 SKIP_INDEXES=0 ./tox_runner.sh .. + @RIAK_TEST_PROTOCOL='pbc' RUN_YZ=1 SKIP_DATATYPES=0 SKIP_INDEXES=0 ./tox_runner.sh .. + @RIAK_TEST_PROTOCOL='http' RUN_YZ=1 SKIP_DATATYPES=0 SKIP_INDEXES=0 ./tox_runner.sh .. test_security: @echo "Testing Riak Python Client (with security)" @../setup.py enable_security --riak-admin=${RIAK_ADMIN} - @RUN_YZ=1 SKIP_INDEXES=0 RUN_SECURITY=1 SKIP_POOL=1 SKIP_RESOLVE=1 RIAK_TEST_HTTP_PORT=18098 ./tox_runner.sh .. + @RIAK_TEST_PROTOCOL='pbc' RUN_YZ=1 SKIP_INDEXES=0 RUN_SECURITY=1 SKIP_POOL=1 SKIP_RESOLVE=1 ./tox_runner.sh .. + @RIAK_TEST_PROTOCOL='http' RUN_YZ=1 SKIP_INDEXES=0 RUN_SECURITY=1 SKIP_POOL=1 SKIP_RESOLVE=1 RIAK_TEST_HTTP_PORT=18098 ./tox_runner.sh .. # These are required to actually build all the Python versions: # * pip install tox diff --git a/riak/tests/__init__.py b/riak/tests/__init__.py index 85599d4b..a7bb4742 100644 --- a/riak/tests/__init__.py +++ b/riak/tests/__init__.py @@ -19,7 +19,7 @@ HOST = os.environ.get('RIAK_TEST_HOST', '127.0.0.1') -PROTOCOL = 'pbc' +PROTOCOL = os.environ.get('RIAK_TEST_PROTOCOL', 'pbc') PB_HOST = os.environ.get('RIAK_TEST_PB_HOST', HOST) PB_PORT = int(os.environ.get('RIAK_TEST_PB_PORT', '8087')) diff --git a/riak/tests/base.py b/riak/tests/base.py index dbd662e0..7ac2dde4 100644 --- a/riak/tests/base.py +++ b/riak/tests/base.py @@ -1,37 +1,13 @@ # -*- coding: utf-8 -*- +import logging +import os import random +import sys from riak.client import RiakClient -from riak.tests import HOST, PROTOCOL,PB_PORT, HTTP_PORT, SECURITY_CREDS +from riak.tests import HOST, PROTOCOL, PB_PORT, HTTP_PORT, SECURITY_CREDS -testrun_search_bucket = 'searchbucket' -testrun_props_bucket = 'propsbucket' -testrun_sibs_bucket = 'sibsbucket' - -def setUpModule(): - - c = RiakClient(host=PB_HOST, http_port=HTTP_PORT, - pb_port=PB_PORT, credentials=SECURITY_CREDS) - - c.bucket(testrun_sibs_bucket).allow_mult = True - - if (not SKIP_SEARCH and not RUN_YZ): - b = c.bucket(testrun_search_bucket) - b.enable_search() - - -def tearDownModule(): - c = RiakClient(host=HTTP_HOST, http_port=HTTP_PORT, - pb_port=PB_PORT, credentials=SECURITY_CREDS) - - c.bucket(testrun_sibs_bucket).clear_properties() - c.bucket(testrun_props_bucket).clear_properties() - - if not SKIP_SEARCH and not RUN_YZ: - b = c.bucket(testrun_search_bucket) - b.clear_properties() - -class BaseTestCase(object): +class IntegrationTestBase(object): host = None pb_port = None @@ -66,6 +42,9 @@ def create_client(self, host=None, http_port=None, pb_port=None, credentials = credentials or SECURITY_CREDS + if self.logging_enabled: + self.logger.debug("RiakClient(protocol='%s', host='%s', pb_port='%d', http_port='%d', credentials='%s', client_args='%s')", protocol, host, pb_port, http_port, credentials, client_args) + return RiakClient(protocol=protocol, host=host, http_port=http_port, @@ -73,15 +52,22 @@ def create_client(self, host=None, http_port=None, pb_port=None, pb_port=pb_port, **client_args) def setUp(self): + self.logging_enabled = False + distutils_debug = os.environ.get('DISTUTILS_DEBUG', '0') + if distutils_debug == '1': + self.logging_enabled = True + self.logger = logging.getLogger() + self.logger.level = logging.DEBUG + self.logging_stream_handler = logging.StreamHandler(sys.stdout) + self.logger.addHandler(self.logging_stream_handler) + self.table_name = 'GeoCheckin' self.bucket_name = self.randname() self.key_name = self.randname() - self.search_bucket = testrun_search_bucket - self.sibs_bucket = testrun_sibs_bucket - self.props_bucket = testrun_props_bucket - # self.yz = testrun_yz - # self.yz_index = testrun_yz_index - # self.yz_mr = testrun_yz_mr self.credentials = SECURITY_CREDS self.client = self.create_client() + def tearDown(self): + if self.logging_enabled: + self.logger.removeHandler(self.logging_stream_handler) + diff --git a/riak/tests/test_2i.py b/riak/tests/test_2i.py index 68b2810a..419bce60 100644 --- a/riak/tests/test_2i.py +++ b/riak/tests/test_2i.py @@ -2,7 +2,7 @@ import platform from riak import RiakError from riak.tests import SKIP_INDEXES -from riak.tests.base import BaseTestCase +from riak.tests.base import IntegrationTestBase if platform.python_version() < '2.7': unittest = __import__('unittest2') @@ -10,7 +10,7 @@ import unittest -class TwoITests(BaseTestCase, unittest.TestCase): +class TwoITests(IntegrationTestBase, unittest.TestCase): def is_2i_supported(self): # Immediate test to see if 2i is even supported w/ the backend try: @@ -219,7 +219,7 @@ def test_secondary_index_invalid_name(self): with self.assertRaises(RiakError): bucket.new('k', 'a').add_index('field1', 'value1') - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_set_index(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I not supported") @@ -237,7 +237,7 @@ def test_set_index(self): obj.set_index('bar2_int', 10) self.assertEqual(set((('bar_int', 3), ('bar2_int', 10))), obj.indexes) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_stream_index(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I not supported") @@ -250,7 +250,7 @@ def test_stream_index(self): self.assertEqual(sorted([o1.key, o2.key, o3.key]), sorted(keys)) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_index_return_terms(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I is not supported") @@ -274,7 +274,7 @@ def test_index_return_terms(self): self.assertEqual([(1002, o2.key), (1003, o3.key), (1004, o4.key)], sorted(spairs)) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_index_pagination(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I is not supported") @@ -309,7 +309,7 @@ def test_index_pagination(self): self.assertEqual(3, pagecount) self.assertEqual([o1.key, o2.key, o3.key, o4.key], presults) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_index_pagination_return_terms(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I is not supported") @@ -334,7 +334,7 @@ def test_index_pagination_return_terms(self): self.assertLessEqual(2, len(results)) self.assertEqual([('val3', o3.key), ('val4', o4.key)], page2) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_index_pagination_stream(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I is not supported") @@ -377,7 +377,7 @@ def test_index_pagination_stream(self): self.assertEqual(3, pagecount) self.assertEqual([o1.key, o2.key, o3.key, o4.key], presults) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_index_pagination_stream_return_terms(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I is not supported") @@ -409,7 +409,7 @@ def test_index_pagination_stream_return_terms(self): self.assertLessEqual(2, len(results)) self.assertEqual([('val3', o3.key), ('val4', o4.key)], results) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_index_eq_query_return_terms(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I is not supported") @@ -419,7 +419,7 @@ def test_index_eq_query_return_terms(self): results = bucket.get_index('field2_int', 1001, return_terms=True) self.assertEqual([(1001, o1.key)], results) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_index_eq_query_stream_return_terms(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I is not supported") @@ -432,7 +432,7 @@ def test_index_eq_query_stream_return_terms(self): self.assertEqual([(1001, o1.key)], results) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_index_timeout(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I is not supported") @@ -451,7 +451,7 @@ def test_index_timeout(self): self.assertEqual([o1.key], bucket.get_index('field1_bin', 'val1', timeout='infinity')) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_index_regex(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I is not supported") @@ -466,7 +466,7 @@ def test_index_regex(self): self.assertEqual([('val2', o2.key)], results) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEX is defined') + @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') def test_index_falsey_endkey_gh378(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I is not supported") diff --git a/riak/tests/test_btypes.py b/riak/tests/test_btypes.py index e1992e04..c1b57f6c 100644 --- a/riak/tests/test_btypes.py +++ b/riak/tests/test_btypes.py @@ -2,7 +2,8 @@ from riak import RiakError, RiakObject from riak.bucket import RiakBucket, BucketType from riak.tests import SKIP_BTYPES -from riak.tests.base import BaseTestCase +from riak.tests.base import IntegrationTestBase +from riak.tests.comparison import Comparison if platform.python_version() < '2.7': unittest = __import__('unittest2') @@ -10,7 +11,7 @@ import unittest -class BucketTypeTests(BaseTestCase, unittest.TestCase): +class BucketTypeTests(IntegrationTestBase, unittest.TestCase, Comparison): def test_btype_init(self): btype = self.client.bucket_type('foo') self.assertIsInstance(btype, BucketType) diff --git a/riak/tests/test_all.py b/riak/tests/test_client.py similarity index 63% rename from riak/tests/test_all.py rename to riak/tests/test_client.py index 385fef7b..a53e7ba6 100644 --- a/riak/tests/test_all.py +++ b/riak/tests/test_client.py @@ -1,28 +1,9 @@ -# -*- coding: utf-8 -*- import platform from six import PY2 from threading import Thread - -from riak import RiakError -from riak.client import RiakClient from riak.riak_object import RiakObject - -from riak.tests.test_yokozuna import YZSearchTests -from riak.tests.test_search import SearchTests, \ - EnableSearchTests, SolrSearchTests -from riak.tests.test_mapreduce import MapReduceAliasTests, \ - ErlangMapReduceTests, JSMapReduceTests, LinkTests, MapReduceStreamTests -from riak.tests.test_kv import BasicKVTests, KVFileTests, \ - BucketPropsTest, CounterTests -from riak.tests.test_2i import TwoITests -from riak.tests.test_btypes import BucketTypeTests -from riak.tests.test_security import SecurityTests -from riak.tests.test_datatypes import DatatypeIntegrationTests -from riak.tests.test_timeseries import TimeseriesTests - -from riak.tests import HOST, PB_HOST, PB_PORT, HTTP_HOST, HTTP_PORT, \ - HAVE_PROTO, DUMMY_HTTP_PORT, DUMMY_PB_PORT, \ - SKIP_SEARCH, RUN_YZ, SECURITY_CREDS, SKIP_POOL +from riak.tests import DUMMY_HTTP_PORT, DUMMY_PB_PORT, SKIP_POOL +from riak.tests.base import IntegrationTestBase if PY2: from Queue import Queue @@ -34,7 +15,15 @@ else: import unittest -class ClientTests(object): +class ClientTests(IntegrationTestBase, unittest.TestCase): + def test_uses_client_id_if_given(self): + if self.protocol == 'pbc': + zero_client_id = "\0\0\0\0" + c = self.create_client(client_id=zero_client_id) + self.assertEqual(zero_client_id, c.client_id) + else: + pass + def test_request_retries(self): # We guess at some ports that will be unused by Riak or # anything else. @@ -213,84 +202,3 @@ def test_pool_close(self): self.client.close() self.assertEqual(len(self.client._http_pool.resources), 0) self.assertEqual(len(self.client._pb_pool.resources), 0) - - -class RiakPbcTransportTestCase(BasicKVTests, - KVFileTests, - BucketPropsTest, - TwoITests, - LinkTests, - ErlangMapReduceTests, - JSMapReduceTests, - MapReduceAliasTests, - MapReduceStreamTests, - EnableSearchTests, - SearchTests, - YZSearchTests, - ClientTests, - CounterTests, - BucketTypeTests, - SecurityTests, - DatatypeIntegrationTests, - unittest.TestCase): - - def setUp(self): - if not HAVE_PROTO: - self.skipTest('protobuf is unavailable') - self.host = PB_HOST - self.pb_port = PB_PORT - self.protocol = 'pbc' - super(RiakPbcTransportTestCase, self).setUp() - - def test_uses_client_id_if_given(self): - zero_client_id = "\0\0\0\0" - c = self.create_client(client_id=zero_client_id) - self.assertEqual(zero_client_id, c.client_id) - - -# NB: no Timeseries support in HTTP -class RiakHttpTransportTestCase(BasicKVTests, - KVFileTests, - BucketPropsTest, - TwoITests, - LinkTests, - ErlangMapReduceTests, - JSMapReduceTests, - MapReduceAliasTests, - MapReduceStreamTests, - EnableSearchTests, - SolrSearchTests, - SearchTests, - YZSearchTests, - ClientTests, - CounterTests, - BucketTypeTests, - SecurityTests, - DatatypeIntegrationTests, - unittest.TestCase): - - def setUp(self): - self.host = HTTP_HOST - self.http_port = HTTP_PORT - self.protocol = 'http' - super(RiakHttpTransportTestCase, self).setUp() - - def test_no_returnbody(self): - bucket = self.client.bucket(self.bucket_name) - o = bucket.new(self.key_name, "bar").store(return_body=False) - self.assertEqual(o.vclock, None) - - def test_too_many_link_headers_shouldnt_break_http(self): - bucket = self.client.bucket(self.bucket_name) - o = bucket.new("lots_of_links", "My god, it's full of links!") - for i in range(0, 300): - link = ("other", "key%d" % i, "next") - o.add_link(link) - - o.store() - stored_object = bucket.get("lots_of_links") - self.assertEqual(len(stored_object.links), 300) - - -if __name__ == '__main__': - unittest.main() diff --git a/riak/tests/test_comparison.py b/riak/tests/test_comparison.py index c21bfee5..6f9e6b9c 100644 --- a/riak/tests/test_comparison.py +++ b/riak/tests/test_comparison.py @@ -2,7 +2,7 @@ import platform from riak.riak_object import RiakObject from riak.bucket import RiakBucket, BucketType -from riak.tests.base import BaseTestCase +from riak.tests.base import IntegrationTestBase if platform.python_version() < '2.7': unittest = __import__('unittest2') @@ -136,7 +136,7 @@ def test_object_valid_key(self): self.assertIsNone(b, 'empty object key not allowed') -class RiakClientComparisonTest(BaseTestCase, unittest.TestCase): +class RiakClientComparisonTest(IntegrationTestBase, unittest.TestCase): def test_client_eq(self): self.protocol = 'http' a = self.create_client(host='host1', http_port=11) diff --git a/riak/tests/test_datatypes.py b/riak/tests/test_datatypes.py index 5e5f0614..b8ca881f 100644 --- a/riak/tests/test_datatypes.py +++ b/riak/tests/test_datatypes.py @@ -3,7 +3,7 @@ from riak import RiakBucket, BucketType, RiakObject import riak.datatypes as datatypes from riak.tests import SKIP_DATATYPES -from riak.tests.base import BaseTestCase +from riak.tests.base import IntegrationTestBase from riak.tests.comparison import Comparison if platform.python_version() < '2.7': @@ -147,7 +147,7 @@ def test_removes_require_context(self): self.assertTrue(dtype.modified) -class DatatypeIntegrationTests(BaseTestCase, unittest.TestCase): +class DatatypeIntegrationTests(IntegrationTestBase, unittest.TestCase, Comparison): @unittest.skipIf(SKIP_DATATYPES, 'SKIP_DATATYPES is set') def test_dt_counter(self): btype = self.client.bucket_type('pytest-counters') diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index b1752e8f..78decef2 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -5,10 +5,10 @@ import copy from time import sleep -from riak import ConflictError, RiakBucket, RiakError +from riak import ConflictError, RiakClient, RiakBucket, RiakError from riak.resolver import default_resolver, last_written_resolver -from riak.tests import SKIP_RESOLVE -from riak.tests.base import BaseTestCase +from riak.tests import SKIP_RESOLVE, HOST, PROTOCOL, PB_PORT, HTTP_PORT, SECURITY_CREDS +from riak.tests.base import IntegrationTestBase from riak.tests.comparison import Comparison try: @@ -31,6 +31,21 @@ test_pickle_loads = pickle.loads +testrun_sibs_bucket = 'sibsbucket' +testrun_props_bucket = 'propsbucket' + +def setUpModule(): + c = RiakClient(protocol=PROTOCOL, host=HOST, http_port=HTTP_PORT, + pb_port=PB_PORT, credentials=SECURITY_CREDS) + c.bucket(testrun_sibs_bucket).allow_mult = True + +def tearDownModule(): + c = RiakClient(protocol=PROTOCOL, host=HOST, http_port=HTTP_PORT, + pb_port=PB_PORT, credentials=SECURITY_CREDS) + c.bucket(testrun_sibs_bucket).clear_properties() + c.bucket(testrun_props_bucket).clear_properties() + + class NotJsonSerializable(object): def __init__(self, *args, **kwargs): @@ -55,7 +70,23 @@ def __eq__(self, other): return True -class BasicKVTests(BaseTestCase, unittest.TestCase, Comparison): +class BasicKVTests(IntegrationTestBase, unittest.TestCase, Comparison): + def test_no_returnbody(self): + bucket = self.client.bucket(self.bucket_name) + o = bucket.new(self.key_name, "bar").store(return_body=False) + self.assertEqual(o.vclock, None) + + def test_many_link_headers_should_work_fine(self): + bucket = self.client.bucket(self.bucket_name) + o = bucket.new("lots_of_links", "My god, it's full of links!") + for i in range(0, 300): + link = ("other", "key%d" % i, "next") + o.add_link(link) + + o.store() + stored_object = bucket.get("lots_of_links") + self.assertEqual(len(stored_object.links), 300) + def test_is_alive(self): self.assertTrue(self.client.is_alive()) @@ -324,19 +355,19 @@ def test_bucket_delete(self): self.assertFalse(obj.exists) def test_set_bucket_properties(self): - bucket = self.client.bucket(self.props_bucket) + bucket = self.client.bucket(testrun_props_bucket) # Test setting allow mult... bucket.allow_mult = True # Test setting nval... bucket.n_val = 1 - bucket2 = self.create_client().bucket(self.props_bucket) + bucket2 = self.create_client().bucket(testrun_props_bucket) self.assertTrue(bucket2.allow_mult) self.assertEqual(bucket2.n_val, 1) # Test setting multiple properties... bucket.set_properties({"allow_mult": False, "n_val": 2}) - bucket3 = self.create_client().bucket(self.props_bucket) + bucket3 = self.create_client().bucket(testrun_props_bucket) self.assertFalse(bucket3.allow_mult) self.assertEqual(bucket3.n_val, 2) @@ -357,7 +388,7 @@ def test_if_none_match(self): def test_siblings(self): # Set up the bucket, clear any existing object... - bucket = self.client.bucket(self.sibs_bucket) + bucket = self.client.bucket(testrun_sibs_bucket) obj = bucket.get(self.key_name) bucket.allow_mult = True @@ -396,7 +427,7 @@ def test_siblings(self): @unittest.skipIf(SKIP_RESOLVE == '1', "skip requested for resolvers test") def test_resolution(self): - bucket = self.client.bucket(self.sibs_bucket) + bucket = self.client.bucket(testrun_sibs_bucket) obj = bucket.get(self.key_name) bucket.allow_mult = True @@ -454,13 +485,13 @@ def max_value_resolver(obj): "skip requested for resolvers test") def test_resolution_default(self): # If no resolver is setup, be sure to resolve to default_resolver - bucket = self.client.bucket(self.sibs_bucket) + bucket = self.client.bucket(testrun_sibs_bucket) self.assertEqual(self.client.resolver, default_resolver) self.assertEqual(bucket.resolver, default_resolver) def test_tombstone_siblings(self): # Set up the bucket, clear any existing object... - bucket = self.client.bucket(self.sibs_bucket) + bucket = self.client.bucket(testrun_sibs_bucket) obj = bucket.get(self.key_name) bucket.allow_mult = True @@ -594,9 +625,9 @@ def generate_siblings(self, original, count=5, delay=None): return vals -class BucketPropsTest(BaseTestCase, unittest.TestCase): +class BucketPropsTest(IntegrationTestBase, unittest.TestCase): def test_rw_settings(self): - bucket = self.client.bucket(self.props_bucket) + bucket = self.client.bucket(testrun_props_bucket) self.assertEqual(bucket.r, "quorum") self.assertEqual(bucket.w, "quorum") self.assertEqual(bucket.dw, "quorum") @@ -621,7 +652,7 @@ def test_rw_settings(self): bucket.clear_properties() def test_primary_quora(self): - bucket = self.client.bucket(self.props_bucket) + bucket = self.client.bucket(testrun_props_bucket) self.assertEqual(bucket.pr, 0) self.assertEqual(bucket.pw, 0) @@ -635,7 +666,7 @@ def test_primary_quora(self): bucket.clear_properties() def test_clear_bucket_properties(self): - bucket = self.client.bucket(self.props_bucket) + bucket = self.client.bucket(testrun_props_bucket) bucket.allow_mult = True self.assertTrue(bucket.allow_mult) bucket.n_val = 1 @@ -647,15 +678,15 @@ def test_clear_bucket_properties(self): self.assertEqual(bucket.n_val, 3) -class KVFileTests(BaseTestCase, unittest.TestCase): +class KVFileTests(IntegrationTestBase, unittest.TestCase): def test_store_binary_object_from_file(self): bucket = self.client.bucket(self.bucket_name) - filepath = os.path.join(os.path.dirname(__file__), 'test_all.py') - obj = bucket.new_from_file(self.key_name, filepath) + obj = bucket.new_from_file(self.key_name, __file__) obj.store() obj = bucket.get(self.key_name) self.assertNotEqual(obj.encoded_data, None) - self.assertEqual(obj.content_type, "text/x-python") + self.assertTrue(obj.content_type == 'text/x-python' or + obj.content_type == 'application/x-python-code') def test_store_binary_object_from_file_should_use_default_mimetype(self): bucket = self.client.bucket(self.bucket_name) @@ -675,7 +706,7 @@ def test_store_binary_object_from_file_should_fail_if_file_not_found(self): self.assertFalse(obj.exists) -class CounterTests(BaseTestCase, unittest.TestCase): +class CounterTests(IntegrationTestBase, unittest.TestCase): def test_counter_requires_allow_mult(self): bucket = self.client.bucket(self.bucket_name) if bucket.allow_mult: @@ -686,7 +717,7 @@ def test_counter_requires_allow_mult(self): bucket.update_counter(self.key_name, 10) def test_counter_ops(self): - bucket = self.client.bucket(self.sibs_bucket) + bucket = self.client.bucket(testrun_sibs_bucket) self.assertTrue(bucket.allow_mult) # Non-existent counter has no value diff --git a/riak/tests/test_mapreduce.py b/riak/tests/test_mapreduce.py index 99ed8a92..e0897f6a 100644 --- a/riak/tests/test_mapreduce.py +++ b/riak/tests/test_mapreduce.py @@ -5,34 +5,29 @@ from six import PY2 from riak.mapreduce import RiakMapReduce -from riak import key_filter, RiakClient, RiakError -from riak.tests import RUN_YZ, PB_HOST, PB_PORT, HTTP_HOST, HTTP_PORT, SECURITY_CREDS -from riak.tests.base import BaseTestCase +from riak import key_filter, RiakError +from riak.tests import RUN_YZ +from riak.tests.base import IntegrationTestBase from riak.tests.test_yokozuna import wait_for_yz_index from riak.tests import RUN_SECURITY -from riak.tests.yz_setup import yzSetUpModule, yzTearDownModule +from riak.tests.yz_setup import yzSetUp, yzTearDown if platform.python_version() < '2.7': unittest = __import__('unittest2') else: import unittest -# Add bucket and type for Search 2.0 -> MapReduce + testrun_yz_mr = {'btype': 'pytest-mr', 'bucket': 'mrbucket', 'index': 'mrbucket'} def setUpModule(): - if RUN_YZ: - c = RiakClient(host=PB_HOST, http_port=HTTP_PORT, - pb_port=PB_PORT, credentials=SECURITY_CREDS) - yzSetUpModule(c, testrun_yz_mr) + yzSetUp(testrun_yz_mr) def tearDownModule(): - if RUN_YZ: - c = RiakClient(host=HTTP_HOST, http_port=HTTP_PORT, - pb_port=PB_PORT, credentials=SECURITY_CREDS) - yzTearDownModule(c, testrun_yz_mr) + yzTearDown(testrun_yz_mr) + -class LinkTests(BaseTestCase, unittest.TestCase): +class LinkTests(IntegrationTestBase, unittest.TestCase): def test_store_and_get_links(self): # Create the object... bucket = self.client.bucket(self.bucket_name) @@ -98,7 +93,7 @@ def test_link_walking(self): self.assertEqual(len(results), 1) -class ErlangMapReduceTests(BaseTestCase, unittest.TestCase): +class ErlangMapReduceTests(IntegrationTestBase, unittest.TestCase): def test_erlang_map_reduce(self): # Create the object... bucket = self.client.bucket(self.bucket_name) @@ -204,7 +199,8 @@ def test_client_exceptional_paths(self): mr.add_key_filter("tokenize", "-", 1) -class JSMapReduceTests(BaseTestCase, unittest.TestCase): +class JSMapReduceTests(IntegrationTestBase, unittest.TestCase): + def test_javascript_source_map(self): # Create the object... bucket = self.client.bucket(self.bucket_name) @@ -564,7 +560,7 @@ def test_mr_search(self): self.assertEqual(result, [100]) -class MapReduceAliasTests(BaseTestCase, unittest.TestCase): +class MapReduceAliasTests(IntegrationTestBase, unittest.TestCase): """This tests the map reduce aliases""" def test_map_values(self): @@ -759,7 +755,7 @@ def test_filter_not_found(self): self.assertEqual(sorted(result), [1, 2]) -class MapReduceStreamTests(BaseTestCase, unittest.TestCase): +class MapReduceStreamTests(IntegrationTestBase, unittest.TestCase): def test_stream_results(self): bucket = self.client.bucket(self.bucket_name) bucket.new('one', data=1).store() diff --git a/riak/tests/test_pool.py b/riak/tests/test_pool.py index 1f0e2f19..edfba3f5 100644 --- a/riak/tests/test_pool.py +++ b/riak/tests/test_pool.py @@ -5,7 +5,7 @@ from riak.transports.pool import Pool, BadResource from random import SystemRandom from time import sleep -from . import SKIP_POOL +from riak.tests import SKIP_POOL from riak.tests.comparison import Comparison if platform.python_version() < '2.7': diff --git a/riak/tests/test_search.py b/riak/tests/test_search.py index 11f9f4c9..799ce8e3 100644 --- a/riak/tests/test_search.py +++ b/riak/tests/test_search.py @@ -1,16 +1,32 @@ # -*- coding: utf-8 -*- from __future__ import print_function import platform -from riak.tests import SKIP_SEARCH -from riak.tests.base import BaseTestCase +from riak import RiakClient +from riak.tests import SKIP_SEARCH, HOST, PROTOCOL, PB_PORT, HTTP_PORT, SECURITY_CREDS +from riak.tests.base import IntegrationTestBase if platform.python_version() < '2.7': unittest = __import__('unittest2') else: import unittest +testrun_search_bucket = 'searchbucket' -class EnableSearchTests(BaseTestCase, unittest.TestCase): +def setUpModule(): + if not SKIP_SEARCH and not RUN_YZ: + c = RiakClient(protocol=PROTOCOL, host=HOST, http_port=HTTP_PORT, + pb_port=PB_PORT, credentials=SECURITY_CREDS) + b = c.bucket(testrun_search_bucket) + b.enable_search() + +def tearDownModule(): + if not SKIP_SEARCH and not RUN_YZ: + c = RiakClient(protocol=PROTOCOL, host=HOST, http_port=HTTP_PORT, + pb_port=PB_PORT, credentials=SECURITY_CREDS) + b = c.bucket(testrun_search_bucket) + b.clear_properties() + +class EnableSearchTests(IntegrationTestBase, unittest.TestCase): @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_bucket_search_enabled(self): bucket = self.client.bucket(self.bucket_name) @@ -18,124 +34,124 @@ def test_bucket_search_enabled(self): @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_enable_search_commit_hook(self): - bucket = self.client.bucket(self.search_bucket) + bucket = self.client.bucket(testrun_search_bucket) bucket.clear_properties() self.assertFalse(self.create_client(). - bucket(self.search_bucket). + bucket(testrun_search_bucket). search_enabled()) bucket.enable_search() self.assertTrue(self.create_client(). - bucket(self.search_bucket). + bucket(testrun_search_bucket). search_enabled()) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_disable_search_commit_hook(self): - bucket = self.client.bucket(self.search_bucket) + bucket = self.client.bucket(testrun_search_bucket) bucket.clear_properties() bucket.enable_search() - self.assertTrue(self.create_client().bucket(self.search_bucket) + self.assertTrue(self.create_client().bucket(testrun_search_bucket) .search_enabled()) bucket.disable_search() - self.assertFalse(self.create_client().bucket(self.search_bucket) + self.assertFalse(self.create_client().bucket(testrun_search_bucket) .search_enabled()) bucket.enable_search() -class SolrSearchTests(BaseTestCase, unittest.TestCase): +class SolrSearchTests(IntegrationTestBase, unittest.TestCase): @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_add_document_to_index(self): - self.client.fulltext_add(self.search_bucket, + self.client.fulltext_add(testrun_search_bucket, [{"id": "doc", "username": "tony"}]) - results = self.client.fulltext_search(self.search_bucket, + results = self.client.fulltext_search(testrun_search_bucket, "username:tony") self.assertEqual("tony", results['docs'][0]['username']) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_add_multiple_documents_to_index(self): self.client.fulltext_add( - self.search_bucket, + testrun_search_bucket, [{"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}]) results = self.client.fulltext_search( - self.search_bucket, "username:russell OR username:dizzy") + testrun_search_bucket, "username:russell OR username:dizzy") self.assertEqual(2, len(results['docs'])) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_id(self): self.client.fulltext_add( - self.search_bucket, + testrun_search_bucket, [{"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}]) - self.client.fulltext_delete(self.search_bucket, docs=["dizzy"]) + self.client.fulltext_delete(testrun_search_bucket, docs=["dizzy"]) results = self.client.fulltext_search( - self.search_bucket, "username:russell OR username:dizzy") + testrun_search_bucket, "username:russell OR username:dizzy") self.assertEqual(1, len(results['docs'])) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_query(self): self.client.fulltext_add( - self.search_bucket, + testrun_search_bucket, [{"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}]) self.client.fulltext_delete( - self.search_bucket, + testrun_search_bucket, queries=["username:dizzy", "username:russell"]) results = self.client.fulltext_search( - self.search_bucket, "username:russell OR username:dizzy") + testrun_search_bucket, "username:russell OR username:dizzy") self.assertEqual(0, len(results['docs'])) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_query_and_id(self): self.client.fulltext_add( - self.search_bucket, + testrun_search_bucket, [{"id": "dizzy", "username": "dizzy"}, {"id": "russell", "username": "russell"}]) self.client.fulltext_delete( - self.search_bucket, + testrun_search_bucket, docs=["dizzy"], queries=["username:russell"]) results = self.client.fulltext_search( - self.search_bucket, + testrun_search_bucket, "username:russell OR username:dizzy") self.assertEqual(0, len(results['docs'])) -class SearchTests(BaseTestCase, unittest.TestCase): +class SearchTests(IntegrationTestBase, unittest.TestCase): @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_solr_search_from_bucket(self): - bucket = self.client.bucket(self.search_bucket) + bucket = self.client.bucket(testrun_search_bucket) bucket.new("user", {"username": "roidrage"}).store() results = bucket.search("username:roidrage") self.assertEqual(1, len(results['docs'])) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_solr_search_with_params_from_bucket(self): - bucket = self.client.bucket(self.search_bucket) + bucket = self.client.bucket(testrun_search_bucket) bucket.new("user", {"username": "roidrage"}).store() results = bucket.search("username:roidrage", wt="xml") self.assertEqual(1, len(results['docs'])) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_solr_search_with_params(self): - bucket = self.client.bucket(self.search_bucket) + bucket = self.client.bucket(testrun_search_bucket) bucket.new("user", {"username": "roidrage"}).store() results = self.client.fulltext_search( - self.search_bucket, + testrun_search_bucket, "username:roidrage", wt="xml") self.assertEqual(1, len(results['docs'])) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_solr_search(self): - bucket = self.client.bucket(self.search_bucket) + bucket = self.client.bucket(testrun_search_bucket) bucket.new("user", {"username": "roidrage"}).store() - results = self.client.fulltext_search(self.search_bucket, + results = self.client.fulltext_search(testrun_search_bucket, "username:roidrage") self.assertEqual(1, len(results["docs"])) @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_search_integration(self): # Create some objects to search across... - bucket = self.client.bucket(self.search_bucket) + bucket = self.client.bucket(testrun_search_bucket) bucket.new("one", {"foo": "one", "bar": "red"}).store() bucket.new("two", {"foo": "two", "bar": "green"}).store() bucket.new("three", {"foo": "three", "bar": "blue"}).store() @@ -143,7 +159,7 @@ def test_search_integration(self): bucket.new("five", {"foo": "five", "bar": "yellow"}).store() # Run some operations... - results = self.client.fulltext_search(self.search_bucket, + results = self.client.fulltext_search(testrun_search_bucket, "foo:one OR foo:two") if (len(results) == 0): print("\n\nNot running test \"testSearchIntegration()\".\n") @@ -154,6 +170,6 @@ def test_search_integration(self): self.assertEqual(len(results['docs']), 2) query = "(foo:one OR foo:two OR foo:three OR foo:four) AND\ (NOT bar:green)" - results = self.client.fulltext_search(self.search_bucket, query) + results = self.client.fulltext_search(testrun_search_bucket, query) self.assertEqual(len(results['docs']), 3) diff --git a/riak/tests/test_security.py b/riak/tests/test_security.py index 065e16ee..f3339e28 100644 --- a/riak/tests/test_security.py +++ b/riak/tests/test_security.py @@ -1,22 +1,29 @@ # -*- coding: utf-8 -*- +import platform import sys + from riak.tests import RUN_SECURITY, SECURITY_USER, SECURITY_PASSWD, \ SECURITY_CACERT, SECURITY_KEY, SECURITY_CERT, SECURITY_REVOKED, \ SECURITY_CERT_USER, SECURITY_CERT_PASSWD, SECURITY_BAD_CERT, \ SECURITY_CREDS, SECURITY_CIPHERS from riak.security import SecurityCreds -from riak.tests.base import BaseTestCase +from riak.tests.base import IntegrationTestBase -if sys.version_info < (2, 7): +if platform.python_version() < '2.7': unittest = __import__('unittest2') else: import unittest -class SecurityTests(BaseTestCase, unittest.TestCase): +class SecurityTests(IntegrationTestBase, unittest.TestCase): @unittest.skipIf(RUN_SECURITY, 'RUN_SECURITY is set') def test_security_disabled(self): - client = self.create_client(credentials=SECURITY_CREDS) + topts = { 'timeout' : 1 } + creds = SecurityCreds(username='foo', + password='bar', + cacert_file=SECURITY_CACERT, + ciphers=SECURITY_CIPHERS) + client = self.create_client(credentials=creds, transport_options=topts) myBucket = client.bucket('test') val1 = "foobar" key1 = myBucket.new('x', data=val1) diff --git a/riak/tests/test_timeseries.py b/riak/tests/test_timeseries.py index 2bfa4585..9e710d7e 100644 --- a/riak/tests/test_timeseries.py +++ b/riak/tests/test_timeseries.py @@ -4,33 +4,32 @@ import sys from riak.tests import SKIP_TIMESERIES -from riak.tests.base import BaseTestCase +from riak.tests.base import IntegrationTestBase if platform.python_version() < '2.7': unittest = __import__('unittest2') else: import unittest -class TimeseriesTests(BaseTestCase, unittest.TestCase): - def setUp(self): - super(TimeseriesTests, self).setUp() - +class TimeseriesTests(IntegrationTestBase, unittest.TestCase): @unittest.skipIf(SKIP_TIMESERIES == '1', "skip requested for timeseries tests") def test_store(self): - now = int(round(time.time() * 1000)) # NB: millis since Jan 1 1970 UTC - fiveMinsInMsec = 5 * 60 * 1000 - fiveMinsAgo = now - fiveMinsInMsec - tenMinsAgo = fiveMinsAgo - fiveMinsInMsec - fifteenMinsAgo = tenMinsAgo - fiveMinsInMsec - twentyMinsAgo = fifteenMinsAgo - fiveMinsInMsec + pass + # TODO RTS-367 + # now = int(round(time.time() * 1000)) # NB: millis since Jan 1 1970 UTC + # fiveMinsInMsec = 5 * 60 * 1000 + # fiveMinsAgo = now - fiveMinsInMsec + # tenMinsAgo = fiveMinsAgo - fiveMinsInMsec + # fifteenMinsAgo = tenMinsAgo - fiveMinsInMsec + # twentyMinsAgo = fifteenMinsAgo - fiveMinsInMsec - table = self.client.table(self.table_name) - measurements = [ - [ 'hash1', 'user2', twentyMinsAgo, 'hurricane', '84.3' ], - [ 'hash1', 'user2', fifteenMinsAgo, 'rain', '79.0' ], - [ 'hash1', 'user2', fiveMinsAgo, 'wind', 50.5 ], - [ 'hash1', 'user2', now, 'snow', 20.1 ] - ] - ts_obj = table.new(measurements) - result = ts_obj.store() - self.assertTrue(result) + # table = self.client.table(self.table_name) + # measurements = [ + # [ 'hash1', 'user2', twentyMinsAgo, 'hurricane', '84.3' ], + # [ 'hash1', 'user2', fifteenMinsAgo, 'rain', '79.0' ], + # [ 'hash1', 'user2', fiveMinsAgo, 'wind', 50.5 ], + # [ 'hash1', 'user2', now, 'snow', 20.1 ] + # ] + # ts_obj = table.new(measurements) + # result = ts_obj.store() + # self.assertTrue(result) diff --git a/riak/tests/test_yokozuna.py b/riak/tests/test_yokozuna.py index 521be2e4..55399aac 100644 --- a/riak/tests/test_yokozuna.py +++ b/riak/tests/test_yokozuna.py @@ -1,33 +1,15 @@ # -*- coding: utf-8 -*- import platform -from riak import RiakClient -from riak.tests import RUN_YZ, PB_HOST, PB_PORT, HTTP_PORT, SECURITY_CREDS -from riak.tests.base import BaseTestCase -from riak.tests.yz_setup import yzSetUpModule, yzTearDownModule +from riak.tests import RUN_YZ +from riak.tests.base import IntegrationTestBase +from riak.tests.comparison import Comparison +from riak.tests.yz_setup import yzSetUp, yzTearDown if platform.python_version() < '2.7': unittest = __import__('unittest2') else: import unittest -# YZ index on bucket of the same name -testrun_yz = {'btype': None, 'bucket': 'yzbucket', 'index': 'yzbucket'} -# YZ index on bucket of a different name -testrun_yz_index = {'btype': None, 'bucket': 'yzindexbucket', 'index': 'yzindex'} - -def setUpModule(): - if RUN_YZ: - c = RiakClient(host=PB_HOST, protocol='pbc', - pb_port=PB_PORT, credentials=SECURITY_CREDS) - yzSetUpModule(c, testrun_yz, testrun_yz_index) - -def tearDownModule(): - if RUN_YZ: - c = RiakClient(host=PB_HOST, protocol='pbc', - pb_port=PB_PORT, credentials=SECURITY_CREDS) - yzTearDownModule(c, testrun_yz, testrun_yz_index) - - def wait_for_yz_index(bucket, key, index=None): """ Wait until Solr index has been updated and a value returns from a query. @@ -40,11 +22,18 @@ def wait_for_yz_index(bucket, key, index=None): while len(bucket.search('_yz_rk:' + key, index=index)['docs']) == 0: pass +# YZ index on bucket of the same name +testrun_yz = {'btype': None, 'bucket': 'yzbucket', 'index': 'yzbucket'} +# YZ index on bucket of a different name +testrun_yz_index = {'btype': None, 'bucket': 'yzindexbucket', 'index': 'yzindex'} + +def setUpModule(): + yzSetUp(testrun_yz, testrun_yz_index) -class YZSearchTests(BaseTestCase, unittest.TestCase): - def setUp(self): - super(YZSearchTests, self).setUp() +def tearDownModule(): + yzTearDown(testrun_yz, testrun_yz_index) +class YZSearchTests(IntegrationTestBase, unittest.TestCase, Comparison): @unittest.skipUnless(RUN_YZ, 'RUN_YZ is undefined') def test_yz_search_from_bucket(self): bucket = self.client.bucket(testrun_yz['bucket']) diff --git a/riak/tests/yz_setup.py b/riak/tests/yz_setup.py index d0e83147..3d4a720e 100644 --- a/riak/tests/yz_setup.py +++ b/riak/tests/yz_setup.py @@ -1,31 +1,42 @@ -from riak import RiakError +import logging -def yzSetUpModule(c, *yzdata): - for yz in yzdata: - c.create_search_index(yz['index'], timeout=30000) - if yz['btype'] is not None: - t = c.bucket_type(yz['btype']) - b = t.bucket(yz['bucket']) - else: - b = c.bucket(yz['bucket']) - # Keep trying to set search bucket property until it succeeds - index_set = False - while not index_set: - try: - b.set_property('search_index', yz['index']) - index_set = True - except RiakError: - pass +from riak import RiakClient, RiakError +from riak.tests import RUN_YZ, PROTOCOL, HOST, PB_PORT, HTTP_PORT, SECURITY_CREDS -def yzTearDownModule(c, *yzdata): - for yz in yzdata: - if yz['btype'] is not None: - t = c.bucket_type(yz['btype']) - b = t.bucket(yz['bucket']) - else: - b = c.bucket(yz['bucket']) - b.set_property('search_index', '_dont_index_') - c.delete_search_index(yz['index']) - for keys in b.stream_keys(): - for key in keys: - b.delete(key) +def yzSetUp(*yzdata): + if RUN_YZ: + c = RiakClient(protocol=PROTOCOL, host=HOST, http_port=HTTP_PORT, + pb_port=PB_PORT, credentials=SECURITY_CREDS) + for yz in yzdata: + logging.debug("yzSetUp: %s", yz) + c.create_search_index(yz['index'], timeout=30000) + if yz['btype'] is not None: + t = c.bucket_type(yz['btype']) + b = t.bucket(yz['bucket']) + else: + b = c.bucket(yz['bucket']) + # Keep trying to set search bucket property until it succeeds + index_set = False + while not index_set: + try: + b.set_property('search_index', yz['index']) + index_set = True + except RiakError: + pass + +def yzTearDown(c, *yzdata): + if RUN_YZ: + c = RiakClient(protocol=PROTOCOL, host=HOST, http_port=HTTP_PORT, + pb_port=PB_PORT, credentials=SECURITY_CREDS) + for yz in yzdata: + logging.debug("yzTearDown: %s", yz) + if yz['btype'] is not None: + t = c.bucket_type(yz['btype']) + b = t.bucket(yz['bucket']) + else: + b = c.bucket(yz['bucket']) + b.set_property('search_index', '_dont_index_') + c.delete_search_index(yz['index']) + for keys in b.stream_keys(): + for key in keys: + b.delete(key) diff --git a/riak/transports/http/__init__.py b/riak/transports/http/__init__.py index c28a0034..c4a19a96 100644 --- a/riak/transports/http/__init__.py +++ b/riak/transports/http/__init__.py @@ -16,6 +16,7 @@ under the License. """ +import logging import socket import select from six import PY2 @@ -86,11 +87,21 @@ def __init__(self, :type timeout: int """ if PY2: + # TODO LRB RTS-367 it appears that pkey_file / cert_file are never set + # in riak/transports/http/connection.py#_connect() method + pkf = pkey_file + if pkf is None and credentials is not None: + pkf = credentials._pkey_file + + cf = cert_file + if cf is None and credentials is not None: + cf = credentials._cert_file + HTTPSConnection.__init__(self, host, port, - key_file=pkey_file, - cert_file=cert_file) + key_file=pkf, + cert_file=cf) else: super(RiakHTTPSConnection, self). \ __init__(host=host, @@ -128,6 +139,8 @@ def connect(self): else: ssl_ctx = configure_ssl_context(self.credentials) host = "riak@" + self.host + if self.timeout is not None: + sock.settimeout(self.timeout) self.sock = ssl.SSLSocket(sock=sock, keyfile=self.credentials.pkey_file, certfile=self.credentials.cert_file, diff --git a/riak/transports/http/connection.py b/riak/transports/http/connection.py index 2912f9b1..e1f570e0 100644 --- a/riak/transports/http/connection.py +++ b/riak/transports/http/connection.py @@ -65,14 +65,21 @@ def _connect(self): """ Use the appropriate connection class; optionally with security. """ + timeout = None + if self._options is not None and 'timeout' in self._options: + timeout = self._options['timeout'] + if self._client._credentials: - self._connection = \ - self._connection_class(self._node.host, - self._node.http_port, - self._client._credentials) + self._connection = self._connection_class( + host=self._node.host, + port=self._node.http_port, + credentials=self._client._credentials, + timeout=timeout) else: - self._connection = self._connection_class(self._node.host, - self._node.http_port) + self._connection = self._connection_class( + host=self._node.host, + port=self._node.http_port, + timeout=timeout) # Forces the population of stats and resources before any # other requests are made. self.server_version diff --git a/riak/transports/http/transport.py b/riak/transports/http/transport.py index 2b197966..c139b3ea 100644 --- a/riak/transports/http/transport.py +++ b/riak/transports/http/transport.py @@ -55,7 +55,7 @@ def __init__(self, node=None, client=None, connection_class=HTTPConnection, client_id=None, - **unused_options): + **options): """ Construct a new HTTP connection to Riak. """ @@ -65,6 +65,7 @@ def __init__(self, node=None, self._node = node self._connection_class = connection_class self._client_id = client_id + self._options = options if not self._client_id: self._client_id = self.make_random_client_id() self._connect() diff --git a/setup.py b/setup.py index dfcce1f3..3e2b84d1 100755 --- a/setup.py +++ b/setup.py @@ -13,9 +13,14 @@ requires.append("pyOpenSSL(>=0.14)") riak_pb_in_pythonpath = False -PYTHONPATH = os.environ.get('PYTHONPATH') -if PYTHONPATH is not None and PYTHONPATH.find('riak_pb/python/lib') != -1: - riak_pb_in_pythonpath = True +os_env_pythonpath = os.environ.get('PYTHONPATH') +if os_env_pythonpath is not None: + for ppath in os_env_pythonpath.split(os.pathsep): + if ppath.find('riak_pb/python/lib') != -1: + riak_pb_messages = os.path.join(ppath, 'riak_pb', 'messages.py') + if os.path.exists(riak_pb_messages): + riak_pb_in_pythonpath = True + break if riak_pb_in_pythonpath: install_requires.append("protobuf ==2.6.1") From 454ba8696d5b7f9052bcb55bb5e615354b449843 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Mon, 26 Oct 2015 17:49:54 -0700 Subject: [PATCH 06/30] Add code to query TS data and encode / decode from PB messages. Tests as well --- README.rst | 8 ++ riak/__init__.py | 26 ---- riak/client/__init__.py | 47 +++---- riak/client/operations.py | 41 +++--- riak/riak_object.py | 20 --- riak/table.py | 44 +++--- riak/tests/base.py | 2 +- riak/tests/test_kv.py | 2 + riak/tests/test_search.py | 2 + riak/tests/test_timeseries.py | 227 +++++++++++++++++++++++++++---- riak/tests/yz_setup.py | 2 + riak/transports/pbc/codec.py | 153 +++++++++++++++++---- riak/transports/pbc/transport.py | 39 +++--- riak/transports/transport.py | 2 +- riak/ts_object.py | 50 +++---- riak/util.py | 18 --- 16 files changed, 449 insertions(+), 234 deletions(-) diff --git a/README.rst b/README.rst index 85468584..94b429b6 100644 --- a/README.rst +++ b/README.rst @@ -234,3 +234,11 @@ To run the tests, then simply .. code-block:: console RUN_SECURITY=1 RIAK_TEST_HTTP_PORT=18098 python setup.py test + +Contributors +-------------------------- + - Rusty Klophaus + - Justin Sheehy + - Jay Baird + - Andy Gross + - Jon Meredith diff --git a/riak/__init__.py b/riak/__init__.py index 415f6660..9e761a91 100644 --- a/riak/__init__.py +++ b/riak/__init__.py @@ -1,34 +1,8 @@ """ -Copyright 2015 Basho Technologies -Copyright 2010 Rusty Klophaus -Copyright 2010 Justin Sheehy -Copyright 2009 Jay Baird - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. ---- The Riak API for Python allows you to connect to a Riak instance, create, modify, and delete Riak objects, add and remove links from Riak objects, run Javascript (and Erlang) based Map/Reduce operations, and run Linkwalking operations. - -See the unit_tests.py file for example usage. - -@author Rusty Klophaus (@rklophaus) (rusty@basho.com) -@author Andy Gross (@argv0) (andy@basho.com) -@author Jon Meredith (@jmeredith) (jmeredith@basho.com) -@author Jay Baird (@skatterbean) (jay@mochimedia.com) """ from riak.riak_error import RiakError, ConflictError diff --git a/riak/client/__init__.py b/riak/client/__init__.py index 002991d8..d623f970 100644 --- a/riak/client/__init__.py +++ b/riak/client/__init__.py @@ -1,24 +1,3 @@ -""" -Copyright 2011 Basho Technologies, Inc. -Copyright 2010 Rusty Klophaus -Copyright 2010 Justin Sheehy -Copyright 2009 Jay Baird - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - try: import simplejson as json except ImportError: @@ -31,6 +10,7 @@ from riak.bucket import RiakBucket, BucketType from riak.mapreduce import RiakMapReduceChain from riak.resolver import default_resolver +from riak.table import Table from riak.transports.http import RiakHttpPool from riak.transports.pbc import RiakPbcPool from riak.security import SecurityCreds @@ -140,6 +120,7 @@ def __init__(self, protocol='pbc', transport_options={}, nodes=None, 'binary/octet-stream': binary_encoder_decoder} self._buckets = WeakValueDictionary() self._bucket_types = WeakValueDictionary() + self._tables = WeakValueDictionary() def _get_protocol(self): return self._protocol @@ -277,12 +258,12 @@ def bucket_type(self, name): not always exist (unlike buckets), but this will always return a :class:`BucketType ` object. - :param name: the bucket name + :param name: the bucket-type name :type name: str :rtype: :class:`BucketType ` """ if not isinstance(name, string_types): - raise TypeError('Bucket name must be a string') + raise TypeError('BucketType name must be a string') if name in self._bucket_types: return self._bucket_types[name] @@ -291,6 +272,26 @@ def bucket_type(self, name): self._bucket_types[name] = btype return btype + def table(self, name): + """ + Gets the table by the specified name. Tables do + not always exist (unlike buckets), but this will always return + a :class:`Table ` object. + + :param name: the table name + :type name: str + :rtype: :class:`Table ` + """ + if not isinstance(name, string_types): + raise TypeError('Table name must be a string') + + if name in self._tables: + return self._tables[name] + else: + table = Table(self, name) + self._tables[name] = table + return table + def close(self): """ Iterate through all of the connections and close each one. diff --git a/riak/client/operations.py b/riak/client/operations.py index 36f6ed1f..f1881fcf 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -1,26 +1,9 @@ -""" -Copyright 2012 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - from riak.client.transport import RiakClientTransport, \ retryable, retryableHttpOnly from riak.client.multiget import multiget from riak.client.index_page import IndexPage from riak.datatypes import TYPES +from riak.table import Table from riak.util import bytes_to_str from six import string_types, PY2 @@ -565,9 +548,31 @@ def ts_put(self, transport, tsobj): :param tsobj: the time series object to store :type tsobj: RiakTsObject + :rtype: boolean """ return transport.ts_put(tsobj) + @retryable + def ts_query(self, transport, table, query, interpolations=None): + """ + ts_query(table, query, interpolations=None) + + Queries time series data in the Riak cluster. + + .. note:: This request is automatically retried :attr:`retries` + times if it fails due to network error. + + :param table: The timeseries table. + :type table: string or :class:`Table ` + :param query: The timeseries query. + :type query: string + :rtype: :class:`TsObject ` + """ + t = table + if isinstance(t, str): + t = Table(self, table) + return transport.ts_query(t, query, interpolations) + @retryable def get(self, transport, robj, r=None, pr=None, timeout=None, basic_quorum=None, notfound_ok=None): diff --git a/riak/riak_object.py b/riak/riak_object.py index 2db8b5e8..ab7dd375 100644 --- a/riak/riak_object.py +++ b/riak/riak_object.py @@ -1,23 +1,3 @@ -""" -Copyright 2012-2013 Basho Technologies -Copyright 2010 Rusty Klophaus -Copyright 2010 Justin Sheehy -Copyright 2009 Jay Baird - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" from riak import ConflictError from riak.content import RiakContent import base64 diff --git a/riak/table.py b/riak/table.py index a739e9bb..10da507c 100644 --- a/riak/table.py +++ b/riak/table.py @@ -1,27 +1,9 @@ -""" -Copyright 2015 Basho Technologies - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" from six import string_types, PY2 - class Table(object): """ - The ``Table`` object allows you to access properties on a Riak table - (bucket type) and query timeseries data. + The ``Table`` object allows you to access properties on a Riak + timeseries table and query timeseries data. """ def __init__(self, client, name): """ @@ -46,6 +28,26 @@ def __init__(self, client, name): self._client = client self.name = name + def __str__(self): + return self.name + + def __repr__(self): + return self.name + + def new(self, rows, columns=None): + """ + A shortcut for manually instantiating a new :class:`~riak.ts_object.TsObject` + + :param rows: An list of lists with timeseries data + :type rows: list + :param columns: An list of Column names and types. Optional. + :type columns: list + :rtype: :class:`~riak.ts_object.TsObject` + """ + from riak.ts_object import TsObject + + return TsObject(self._client, self, rows, columns) + def query(self, query, interpolations=None): """ Queries a timeseries table. @@ -54,4 +56,4 @@ def query(self, query, interpolations=None): :type query: string :rtype: :class:`TsObject ` """ - return self.client.ts_query(query, interpolations) + return self.client.ts_query(self, query, interpolations) diff --git a/riak/tests/base.py b/riak/tests/base.py index 7ac2dde4..0fb4317f 100644 --- a/riak/tests/base.py +++ b/riak/tests/base.py @@ -61,13 +61,13 @@ def setUp(self): self.logging_stream_handler = logging.StreamHandler(sys.stdout) self.logger.addHandler(self.logging_stream_handler) - self.table_name = 'GeoCheckin' self.bucket_name = self.randname() self.key_name = self.randname() self.credentials = SECURITY_CREDS self.client = self.create_client() def tearDown(self): + self.client.close() if self.logging_enabled: self.logger.removeHandler(self.logging_stream_handler) diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 78decef2..3ba4eedc 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -38,12 +38,14 @@ def setUpModule(): c = RiakClient(protocol=PROTOCOL, host=HOST, http_port=HTTP_PORT, pb_port=PB_PORT, credentials=SECURITY_CREDS) c.bucket(testrun_sibs_bucket).allow_mult = True + c.close() def tearDownModule(): c = RiakClient(protocol=PROTOCOL, host=HOST, http_port=HTTP_PORT, pb_port=PB_PORT, credentials=SECURITY_CREDS) c.bucket(testrun_sibs_bucket).clear_properties() c.bucket(testrun_props_bucket).clear_properties() + c.close() class NotJsonSerializable(object): diff --git a/riak/tests/test_search.py b/riak/tests/test_search.py index 799ce8e3..ebf13730 100644 --- a/riak/tests/test_search.py +++ b/riak/tests/test_search.py @@ -18,6 +18,7 @@ def setUpModule(): pb_port=PB_PORT, credentials=SECURITY_CREDS) b = c.bucket(testrun_search_bucket) b.enable_search() + c.close() def tearDownModule(): if not SKIP_SEARCH and not RUN_YZ: @@ -25,6 +26,7 @@ def tearDownModule(): pb_port=PB_PORT, credentials=SECURITY_CREDS) b = c.bucket(testrun_search_bucket) b.clear_properties() + c.close() class EnableSearchTests(IntegrationTestBase, unittest.TestCase): @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') diff --git a/riak/tests/test_timeseries.py b/riak/tests/test_timeseries.py index 9e710d7e..34aa23b9 100644 --- a/riak/tests/test_timeseries.py +++ b/riak/tests/test_timeseries.py @@ -1,9 +1,17 @@ # -*- coding: utf-8 -*- +import datetime +import os import platform -import time +import riak_pb import sys +import time -from riak.tests import SKIP_TIMESERIES +from riak.table import Table +from riak.ts_object import TsObject +from riak.transports.pbc.codec import RiakPbcCodec +from riak import RiakClient +from riak.util import str_to_bytes +from riak.tests import SKIP_TIMESERIES, HOST, PROTOCOL, PB_PORT, HTTP_PORT, SECURITY_CREDS from riak.tests.base import IntegrationTestBase if platform.python_version() < '2.7': @@ -11,25 +19,198 @@ else: import unittest +table_name = 'GeoCheckin' + +bd0 = os.urandom(16) +bd1 = os.urandom(16) + +fiveMins = datetime.timedelta(0, 300) +ts0 = datetime.datetime(2015, 1, 1, 12, 0, 0) +ts1 = ts0 + fiveMins + +s = [ 'foo', 'bar', 'baz' ] +m = { + 'foo': 'foo', + 'bar': 'bar', + 'baz': 'baz', + 'set': s +} +sj = ['"foo"', '"bar"', '"baz"'] +mj = '{"baz": "baz", "set": ["foo", "bar", "baz"], "foo": "foo", "bar": "bar"}' + +class TimeseriesUnitTests(unittest.TestCase): + def setUp(self): + self.c = RiakPbcCodec() + self.ts0ms = self.c._unix_time_millis(ts0) + self.ts1ms = self.c._unix_time_millis(ts1) + self.rows = [ + [ bd0, 0, 1.2, ts0, True, s, m ], + [ bd1, 3, 4.5, ts1, False, s, m ] + ] + self.table = Table(None, 'test-table') + + def test_encode_data(self): + tsobj = TsObject(None, self.table, self.rows, None) + ts_put_req = riak_pb.TsPutReq() + self.c._encode_timeseries(tsobj, ts_put_req) + + # NB: expected, actual + self.assertEqual(len(self.rows), len(ts_put_req.rows)) + + r0 = ts_put_req.rows[0] + self.assertEqual(r0.cells[0].binary_value, self.rows[0][0]) + self.assertEqual(r0.cells[1].integer_value, self.rows[0][1]) + self.assertEqual(r0.cells[2].double_value, self.rows[0][2]) + self.assertEqual(r0.cells[3].timestamp_value, self.ts0ms) + self.assertEqual(r0.cells[4].boolean_value, self.rows[0][4]) + self.assertEqual(r0.cells[5].set_value, sj) + self.assertEqual(r0.cells[6].map_value, mj) + + r1 = ts_put_req.rows[1] + self.assertEqual(r1.cells[0].binary_value, self.rows[1][0]) + self.assertEqual(r1.cells[1].integer_value, self.rows[1][1]) + self.assertEqual(r1.cells[2].double_value, self.rows[1][2]) + self.assertEqual(r1.cells[3].timestamp_value, self.ts1ms) + self.assertEqual(r1.cells[4].boolean_value, self.rows[1][4]) + self.assertEqual(r1.cells[5].set_value, sj) + self.assertEqual(r1.cells[6].map_value, mj) + + def test_decode_data(self): + tqr = riak_pb.TsQueryResp() + + c0 = tqr.columns.add() + c0.name = str_to_bytes('col_binary') + c0.type = riak_pb.TsColumnType.Value('BINARY') + c1 = tqr.columns.add() + c1.name = str_to_bytes('col_integer') + c1.type = riak_pb.TsColumnType.Value('INTEGER') + c2 = tqr.columns.add() + c2.name = str_to_bytes('col_double') + c2.type = riak_pb.TsColumnType.Value('FLOAT') + c3 = tqr.columns.add() + c3.name = str_to_bytes('col_timestamp') + c3.type = riak_pb.TsColumnType.Value('TIMESTAMP') + c4 = tqr.columns.add() + c4.name = str_to_bytes('col_boolean') + c4.type = riak_pb.TsColumnType.Value('BOOLEAN') + c5 = tqr.columns.add() + c5.name = str_to_bytes('col_set') + c5.type = riak_pb.TsColumnType.Value('SET') + c6 = tqr.columns.add() + c6.name = str_to_bytes('col_map') + c6.type = riak_pb.TsColumnType.Value('MAP') + + r0 = tqr.rows.add() + r0c0 = r0.cells.add() + r0c0.binary_value = self.rows[0][0] + r0c1 = r0.cells.add() + r0c1.integer_value = self.rows[0][1] + r0c2 = r0.cells.add() + r0c2.double_value = self.rows[0][2] + r0c3 = r0.cells.add() + r0c3.timestamp_value = self.ts0ms + r0c4 = r0.cells.add() + r0c4.boolean_value = self.rows[0][4] + r0c5 = r0.cells.add() + for j in sj: + r0c5.set_value.append(j) + r0c6 = r0.cells.add() + r0c6.map_value = str_to_bytes(mj) + + r1 = tqr.rows.add() + r1c0 = r1.cells.add() + r1c0.binary_value = self.rows[1][0] + r1c1 = r1.cells.add() + r1c1.integer_value = self.rows[1][1] + r1c2 = r1.cells.add() + r1c2.double_value = self.rows[1][2] + r1c3 = r1.cells.add() + r1c3.timestamp_value = self.ts1ms + r1c4 = r1.cells.add() + r1c4.boolean_value = self.rows[1][4] + r1c5 = r1.cells.add() + for j in sj: + r1c5.set_value.append(j) + r1c6 = r1.cells.add() + r1c6.map_value = str_to_bytes(mj) + + tsobj = TsObject(None, self.table, [], []) + c = RiakPbcCodec() + c._decode_timeseries(tqr, tsobj) + + self.assertEqual(len(self.rows), len(tsobj.rows)) + self.assertEqual(len(tqr.columns), len(tsobj.columns)) + + c = tsobj.columns + self.assertEqual(c[0][0], 'col_binary') + self.assertEqual(c[0][1], riak_pb.TsColumnType.Value('BINARY')) + self.assertEqual(c[1][0], 'col_integer') + self.assertEqual(c[1][1], riak_pb.TsColumnType.Value('INTEGER')) + self.assertEqual(c[2][0], 'col_double') + self.assertEqual(c[2][1], riak_pb.TsColumnType.Value('FLOAT')) + self.assertEqual(c[3][0], 'col_timestamp') + self.assertEqual(c[3][1], riak_pb.TsColumnType.Value('TIMESTAMP')) + self.assertEqual(c[4][0], 'col_boolean') + self.assertEqual(c[4][1], riak_pb.TsColumnType.Value('BOOLEAN')) + self.assertEqual(c[5][0], 'col_set') + self.assertEqual(c[5][1], riak_pb.TsColumnType.Value('SET')) + self.assertEqual(c[6][0], 'col_map') + self.assertEqual(c[6][1], riak_pb.TsColumnType.Value('MAP')) + + r0 = tsobj.rows[0] + self.assertEqual(r0[0], self.rows[0][0]) + self.assertEqual(r0[1], self.rows[0][1]) + self.assertEqual(r0[2], self.rows[0][2]) + self.assertEqual(r0[3], ts0) + self.assertEqual(r0[4], self.rows[0][4]) + self.assertEqual(r0[5], s) + self.assertEqual(r0[6], m) + + r1 = tsobj.rows[1] + self.assertEqual(r1[0], self.rows[1][0]) + self.assertEqual(r1[1], self.rows[1][1]) + self.assertEqual(r1[2], self.rows[1][2]) + self.assertEqual(r1[3], ts1) + self.assertEqual(r1[4], self.rows[1][4]) + self.assertEqual(r1[5], s) + self.assertEqual(r1[6], m) + +@unittest.skipIf(SKIP_TIMESERIES == 1, "skip requested for timeseries tests") class TimeseriesTests(IntegrationTestBase, unittest.TestCase): - @unittest.skipIf(SKIP_TIMESERIES == '1', "skip requested for timeseries tests") - def test_store(self): - pass - # TODO RTS-367 - # now = int(round(time.time() * 1000)) # NB: millis since Jan 1 1970 UTC - # fiveMinsInMsec = 5 * 60 * 1000 - # fiveMinsAgo = now - fiveMinsInMsec - # tenMinsAgo = fiveMinsAgo - fiveMinsInMsec - # fifteenMinsAgo = tenMinsAgo - fiveMinsInMsec - # twentyMinsAgo = fifteenMinsAgo - fiveMinsInMsec - - # table = self.client.table(self.table_name) - # measurements = [ - # [ 'hash1', 'user2', twentyMinsAgo, 'hurricane', '84.3' ], - # [ 'hash1', 'user2', fifteenMinsAgo, 'rain', '79.0' ], - # [ 'hash1', 'user2', fiveMinsAgo, 'wind', 50.5 ], - # [ 'hash1', 'user2', now, 'snow', 20.1 ] - # ] - # ts_obj = table.new(measurements) - # result = ts_obj.store() - # self.assertTrue(result) + @classmethod + def setUpClass(cls): + cls.now = datetime.datetime.utcfromtimestamp(144379690) + fiveMinsAgo = cls.now - fiveMins + tenMinsAgo = fiveMinsAgo - fiveMins + fifteenMinsAgo = tenMinsAgo - fiveMins + twentyMinsAgo = fifteenMinsAgo - fiveMins + + client = RiakClient(protocol=PROTOCOL, host=HOST, http_port=HTTP_PORT, + pb_port=PB_PORT, credentials=SECURITY_CREDS) + table = client.table(table_name) + rows = [ + [ 'hash1', 'user2', twentyMinsAgo, 'hurricane', None ], + [ 'hash1', 'user2', fifteenMinsAgo, 'rain', 79.0 ], + [ 'hash1', 'user2', fiveMinsAgo, 'wind', 50.5 ], + [ 'hash1', 'user2', cls.now, 'snow', 20.1 ] + ] + ts_obj = table.new(rows) + result = ts_obj.store() + + codec = RiakPbcCodec() + cls.nowMsec = codec._unix_time_millis(cls.now) + cls.tenMinsAgoMsec = codec._unix_time_millis(tenMinsAgo) + client.close() + + # TODO RTS-367 ts_query test. Ensure that 'None' comes back, somehow + def test_query_that_returns_no_data(self): + query = "select * from {} where time > 0 and time < 10 and user = 'user1'".format(table_name) + ts_obj = self.client.ts_query('GeoCheckin', query) + self.assertEqual(len(ts_obj.columns), 0) + self.assertEqual(len(ts_obj.rows), 0) + + def test_query_that_matches_some_data(self): + query = "select * from {} where time > {} and time < {} and user = 'user2'".format(table_name, self.tenMinsAgoMsec, self.nowMsec); + ts_obj = self.client.ts_query('GeoCheckin', query) + self.assertEqual(len(ts_obj.columns), 5) + self.assertEqual(len(ts_obj.rows), 1) diff --git a/riak/tests/yz_setup.py b/riak/tests/yz_setup.py index 3d4a720e..f7a88efa 100644 --- a/riak/tests/yz_setup.py +++ b/riak/tests/yz_setup.py @@ -23,6 +23,7 @@ def yzSetUp(*yzdata): index_set = True except RiakError: pass + c.close() def yzTearDown(c, *yzdata): if RUN_YZ: @@ -40,3 +41,4 @@ def yzTearDown(c, *yzdata): for keys in b.stream_keys(): for key in keys: b.delete(key) + c.close() diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index e8b47420..d1d7a3af 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -1,27 +1,15 @@ -""" -Copyright 2012 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" +import datetime +import json +import logging import riak_pb + from riak import RiakError from riak.content import RiakContent from riak.util import decode_index_value, str_to_bytes, bytes_to_str from riak.multidict import MultiDict from six import string_types, PY2 +epoch = datetime.datetime.utcfromtimestamp(0) def _invert(d): out = {} @@ -87,6 +75,12 @@ def __init__(self, **unused_args): raise NotImplementedError("this transport is not available") super(RiakPbcCodec, self).__init__(**unused_args) + def _unix_time_millis(self, dt): + return int((dt - epoch).total_seconds() * 1000.0) + + def _datetime_from_unix_time_millis(self, ut): + return datetime.datetime.utcfromtimestamp(ut / 1000.0) + def _encode_quorum(self, rw): """ Converts a symbolic quorum value into its on-the-wire @@ -627,25 +621,126 @@ def _encode_map_update(self, dtype, msg, op): def _encode_timeseries(self, tsobj, ts_put_req): """ Fills an TsPutReq message with the appropriate data and - metadata from a RiakTsObject. + metadata from a TsObject. - :param tsobj: a RiakTsObject - :type tsobj: RiakTsObject + :param tsobj: a TsObject + :type tsobj: TsObject :param ts_put_req: the protobuf message to fill :type ts_put_req: riak_pb.TsPutReq """ - ts_put_req.table = str_to_bytes(tsobj.table) - # TODO RTS-367 columns / rows - # if tsobj.columns: - # if tsobj.rows: - # else: - # raise RiakError("RiakTsObject requires rows") + ts_put_req.table = str_to_bytes(tsobj.table.name) + if tsobj.columns: + raise NotImplementedError("columns are not implemented yet") + if tsobj.rows and isinstance(tsobj.rows, list): + for row in tsobj.rows: + tsr = ts_put_req.rows.add() # NB: type riak_pb.TsRow + if not isinstance(row, list): + raise RiakError("TsObject row must be a list of values") + for cell in row: + tsc = tsr.cells.add() # NB: type riak_pb.TsCell + if cell is not None: + if isinstance(cell, bytes) or isinstance(cell, bytearray): + logging.debug("cell -> binary_value: '%s'", cell) + tsc.binary_value = cell + elif isinstance(cell, datetime.datetime): + tsc.timestamp_value = self._unix_time_millis(cell) + logging.debug("cell -> timestamp: '%s', timestamp_value '%d'", + cell, tsc.timestamp_value) + elif isinstance(cell, list): + for c in cell: + j = json.dumps(c) + logging.debug("cell -> set_value: '%s'", j) + tsc.set_value.append(str_to_bytes(j)) + elif isinstance(cell, bool): + logging.debug("cell -> boolean: '%s'", cell) + tsc.boolean_value = cell + elif isinstance(cell, str): + logging.debug("cell -> str: '%s'", cell) + tsc.binary_value = str_to_bytes(cell) + elif isinstance(cell, int) or isinstance(cell, long): + logging.debug("cell -> int/long: '%s'", cell) + tsc.integer_value = cell + elif isinstance(cell, float): + logging.debug("cell -> float: '%s'", cell) + tsc.double_value = cell + elif isinstance(cell, dict): + logging.debug("cell -> dict: '%s'", cell) + j = json.dumps(cell) + tsc.map_value = str_to_bytes(j) + else: + t = type(cell) + raise RiakError("can't serialize type '{}', value '{}'".format(t, cell)) + else: + raise RiakError("TsObject requires a list of rows") - def _decode_timeseries(self, ts_put_resp, tsobj): + def _decode_timeseries(self, ts_query_rsp, tsobj): """ - TODO RTS-367 + Fills an TsObject with the appropriate data and + metadata from a TsQueryResp. + + :param ts_query_rsp: the protobuf message from which to process data + :type ts_query_rsp: riak_pb.TsQueryRsp + :param tsobj: a TsObject + :type tsobj: TsObject """ - raise NotImplementedError + if not isinstance(ts_query_rsp, riak_pb.TsQueryResp): + raise RiakError("expected riak_pb.TsQueryResp") + + if tsobj.columns is not None: + for ts_col in ts_query_rsp.columns: + col_name = bytes_to_str(ts_col.name) + col_type = ts_col.type + col = (col_name, col_type) + logging.debug("column: '%s'", col) + tsobj.columns.append(col) + + for ts_row in ts_query_rsp.rows: + tsobj.rows.append(self._decode_timeseries_row(ts_row, ts_query_rsp.columns)) + + def _decode_timeseries_row(self, ts_row, ts_columns): + """ + Decodes a TsRow into a list + + :param ts_row: the protobuf TsRow to decode. + :type ts_row: riak_pb.TsRow + :param ts_columns: the protobuf TsColumn data to help decode. + :type ts_columns: list + :rtype list + """ + row = [] + for i, ts_cell in enumerate(ts_row.cells): + ts_col = ts_columns[i] + if ts_col.type == riak_pb.TsColumnType.Value('BINARY'): + logging.debug("ts_cell.binary_value: '%s'", ts_cell.binary_value) + row.append(ts_cell.binary_value) + elif ts_col.type == riak_pb.TsColumnType.Value('INTEGER'): + logging.debug("ts_cell.integer_value: '%s'", ts_cell.integer_value) + row.append(ts_cell.integer_value) + elif ts_col.type == riak_pb.TsColumnType.Value('FLOAT'): + logging.debug("ts_cell.double_value: '%s'", ts_cell.double_value) + row.append(ts_cell.double_value) + elif ts_col.type == riak_pb.TsColumnType.Value('TIMESTAMP'): + dt = self._datetime_from_unix_time_millis(ts_cell.timestamp_value) + logging.debug("ts_cell.timestamp_value: '%s', datetime: '%s'", + ts_cell.timestamp_value, dt) + row.append(dt) + elif ts_col.type == riak_pb.TsColumnType.Value('BOOLEAN'): + logging.debug("ts_cell.boolean_value: '%s'", ts_cell.boolean_value) + row.append(ts_cell.boolean_value) + elif ts_col.type == riak_pb.TsColumnType.Value('SET'): + logging.debug("ts_cell.set_value: '%s'", ts_cell.set_value) + s = [] + for sv in ts_cell.set_value: + sj = bytes_to_str(sv) + s.append(json.loads(sj)) + row.append(s) + elif ts_col.type == riak_pb.TsColumnType.Value('MAP'): + logging.debug("ts_cell.map_value: '%s'", ts_cell.map_value) + mj = bytes_to_str(ts_cell.map_value) + row.append(json.loads(mj)) + else: + row.append(None) + return row def _decode_preflist(self, item): """ diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 5e3e91c8..8a81b26c 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -1,28 +1,8 @@ -""" -Copyright 2015 Basho Technologies, Inc. -Copyright 2010 Rusty Klophaus -Copyright 2010 Justin Sheehy -Copyright 2009 Jay Baird - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - import riak_pb from riak import RiakError from riak.transports.transport import RiakTransport from riak.riak_object import VClock +from riak.ts_object import TsObject from riak.util import decode_index_value, str_to_bytes, bytes_to_str from riak.transports.pbc.connection import RiakPbcConnection from riak.transports.pbc.stream import (RiakPbcKeyStream, @@ -80,8 +60,10 @@ MSG_CODE_DT_FETCH_RESP, MSG_CODE_DT_UPDATE_REQ, MSG_CODE_DT_UPDATE_RESP, - # MSG_CODE_TS_PUT_REQ, - # MSG_CODE_TS_PUT_RESP + MSG_CODE_TS_PUT_REQ, + MSG_CODE_TS_PUT_RESP, + MSG_CODE_TS_QUERY_REQ, + MSG_CODE_TS_QUERY_RESP ) @@ -248,6 +230,17 @@ def ts_put(self, tsobj): elif not robj.key: raise RiakError("missing response object") + def ts_query(self, table, query, interpolations=None): + req = riak_pb.TsQueryReq() + req.query.base = bytes_to_str(query) + + msg_code, ts_query_resp = self._request(MSG_CODE_TS_QUERY_REQ, req, + MSG_CODE_TS_QUERY_RESP) + + tsobj = TsObject(self._client, table, [], []) + self._decode_timeseries(ts_query_resp, tsobj) + return tsobj + def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None, timeout=None): req = riak_pb.RpbDelReq() diff --git a/riak/transports/transport.py b/riak/transports/transport.py index d742d9dd..5e2e5f3e 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -98,7 +98,7 @@ def ts_put(self, tsobj): """ raise NotImplementedError - def ts_query(self, query, interpolations=None): + def ts_query(self, table, query, interpolations=None): """ Query timeseries data. """ diff --git a/riak/ts_object.py b/riak/ts_object.py index 2b855f2e..f1e4f028 100644 --- a/riak/ts_object.py +++ b/riak/ts_object.py @@ -1,30 +1,12 @@ -""" -Copyright 2015 Basho Technologies +from riak import RiakError +from riak.table import Table -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - -# TODO RTS-367 -# Should the table parameter be its own object that has a query method on it? -# Like Bucket? class TsObject(object): """ - The TsObject holds meta information about Timeseries data, - plus the data itself. + The TsObject holds information about Timeseries data, plus the data + itself. """ - def __init__(self, client, table, rows, columns=None): + def __init__(self, client, table, rows=[], columns=[]): """ Construct a new TsObject. @@ -32,23 +14,29 @@ def __init__(self, client, table, rows, columns=None): :type client: :class:`RiakClient ` :param table: The table for the timeseries data as a Table object. :type table: :class:`Table` - :param rows: An array of arrays with timeseries data - :type rows: array - :param columns: An array Column names and types. Optional. - :type columns: array + :param rows: An list of lists with timeseries data + :type rows: list + :param columns: An list of Column names and types. Optional. + :type columns: list """ - if table is None or len(table) == 0: - raise ValueError('Table must either be a non-empty string.') + if not isinstance(table, Table): + raise ValueError('table must be an instance of Table.') self.client = client self.table = table - # TODO RTS-367 rows, columns + + self.rows = rows + if not isinstance(self.rows, list): + raise RiakError("TsObject requires a list of rows") + + self.columns = columns + if self.columns is not None and not isinstance(self.columns, list): + raise RiakError("TsObject columns must be a list") def store(self): """ Store the timeseries data in Riak. :rtype: boolean """ - return self.client.ts_put(self) diff --git a/riak/util.py b/riak/util.py index 9a5b5a14..5dc3e61a 100644 --- a/riak/util.py +++ b/riak/util.py @@ -1,21 +1,3 @@ -""" -Copyright 2014 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - from __future__ import print_function import warnings from collections import Mapping From 4551e14f57c7877b6f2cba74b654ca7e9d894d65 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Mon, 26 Oct 2015 18:02:25 -0700 Subject: [PATCH 07/30] Ensure that RiakClient objects created during tests are closed --- riak/tests/test_client.py | 5 +++++ riak/tests/test_comparison.py | 8 ++++++++ riak/tests/test_kv.py | 10 ++++++++-- riak/tests/test_search.py | 29 +++++++++++++++++++---------- riak/tests/test_security.py | 10 ++++++++++ 5 files changed, 50 insertions(+), 12 deletions(-) diff --git a/riak/tests/test_client.py b/riak/tests/test_client.py index a53e7ba6..be105502 100644 --- a/riak/tests/test_client.py +++ b/riak/tests/test_client.py @@ -21,6 +21,7 @@ def test_uses_client_id_if_given(self): zero_client_id = "\0\0\0\0" c = self.create_client(client_id=zero_client_id) self.assertEqual(zero_client_id, c.client_id) + c.close() else: pass @@ -33,6 +34,7 @@ def test_request_retries(self): # If retries are exhausted, the final result should also be an # error. self.assertRaises(IOError, client.ping) + client.close() def test_request_retries_configurable(self): # We guess at some ports that will be unused by Riak or @@ -61,6 +63,7 @@ def _target(): with client.retry_count(5): self.assertEqual(5, client.retries) self.assertRaises(IOError, client.ping) + client.close() def test_timeout_validation(self): bucket = self.client.bucket(self.bucket_name) @@ -147,6 +150,7 @@ def test_multiget_errors(self): self.assertIsInstance(failure[3], StandardError) # noqa else: self.assertIsInstance(failure[3], Exception) + client.close() def test_multiget_notfounds(self): """ @@ -186,6 +190,7 @@ def test_multiget_pool_size(self): self.assertEqual(obj.key, obj.encoded_data) else: self.assertEqual(obj.key, obj.data) + client.close() @unittest.skipIf(SKIP_POOL, 'SKIP_POOL is set') def test_pool_close(self): diff --git a/riak/tests/test_comparison.py b/riak/tests/test_comparison.py index 6f9e6b9c..446bc031 100644 --- a/riak/tests/test_comparison.py +++ b/riak/tests/test_comparison.py @@ -142,6 +142,8 @@ def test_client_eq(self): a = self.create_client(host='host1', http_port=11) b = self.create_client(host='host1', http_port=11) self.assertEqual(a, b) + a.close() + b.close() def test_client_nq(self): self.protocol = 'http' @@ -150,6 +152,9 @@ def test_client_nq(self): c = self.create_client(host='host1', http_port=12) self.assertNotEqual(a, b, 'matched with different hosts') self.assertNotEqual(a, c, 'matched with different ports') + a.close() + b.close() + c.close() def test_client_hash(self): self.protocol = 'http' @@ -158,6 +163,9 @@ def test_client_hash(self): c = self.create_client(host='host2', http_port=11) self.assertEqual(hash(a), hash(b), 'same object has different hashes') self.assertNotEqual(hash(a), hash(c), 'different object has same hash') + a.close() + b.close() + c.close() if __name__ == '__main__': unittest.main() diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 3ba4eedc..75bb69f2 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -363,16 +363,22 @@ def test_set_bucket_properties(self): # Test setting nval... bucket.n_val = 1 - bucket2 = self.create_client().bucket(testrun_props_bucket) + c2 = self.create_client() + bucket2 = c2.bucket(testrun_props_bucket) self.assertTrue(bucket2.allow_mult) self.assertEqual(bucket2.n_val, 1) # Test setting multiple properties... bucket.set_properties({"allow_mult": False, "n_val": 2}) - bucket3 = self.create_client().bucket(testrun_props_bucket) + c3 = self.create_client() + bucket3 = c3.bucket(testrun_props_bucket) self.assertFalse(bucket3.allow_mult) self.assertEqual(bucket3.n_val, 2) + # clean up! + c2.close() + c3.close() + def test_if_none_match(self): bucket = self.client.bucket(self.bucket_name) obj = bucket.get(self.key_name) diff --git a/riak/tests/test_search.py b/riak/tests/test_search.py index ebf13730..3050de34 100644 --- a/riak/tests/test_search.py +++ b/riak/tests/test_search.py @@ -38,24 +38,33 @@ def test_bucket_search_enabled(self): def test_enable_search_commit_hook(self): bucket = self.client.bucket(testrun_search_bucket) bucket.clear_properties() - self.assertFalse(self.create_client(). - bucket(testrun_search_bucket). - search_enabled()) + + c = self.create_client() + self.assertFalse(c.bucket(testrun_search_bucket).search_enabled()) + c.close() + bucket.enable_search() - self.assertTrue(self.create_client(). - bucket(testrun_search_bucket). - search_enabled()) + + c = self.create_client() + self.assertTrue(c.bucket(testrun_search_bucket).search_enabled()) + c.close() @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_disable_search_commit_hook(self): bucket = self.client.bucket(testrun_search_bucket) bucket.clear_properties() bucket.enable_search() - self.assertTrue(self.create_client().bucket(testrun_search_bucket) - .search_enabled()) + + c = self.create_client() + self.assertTrue(c.bucket(testrun_search_bucket).search_enabled()) + c.close() + bucket.disable_search() - self.assertFalse(self.create_client().bucket(testrun_search_bucket) - .search_enabled()) + + c = self.create_client() + self.assertFalse(c.bucket(testrun_search_bucket).search_enabled()) + c.close() + bucket.enable_search() diff --git a/riak/tests/test_security.py b/riak/tests/test_security.py index f3339e28..0c8a5a49 100644 --- a/riak/tests/test_security.py +++ b/riak/tests/test_security.py @@ -29,6 +29,7 @@ def test_security_disabled(self): key1 = myBucket.new('x', data=val1) with self.assertRaises(Exception): key1.store() + client.close() @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') def test_security_basic_connection(self): @@ -47,6 +48,7 @@ def test_security_bad_user(self): client = self.create_client(credentials=creds) with self.assertRaises(Exception): client.get_buckets() + client.close() @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') def test_security_bad_password(self): @@ -57,6 +59,7 @@ def test_security_bad_password(self): client = self.create_client(credentials=creds) with self.assertRaises(Exception): client.get_buckets() + client.close() @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') def test_security_invalid_cert(self): @@ -67,6 +70,7 @@ def test_security_invalid_cert(self): client = self.create_client(credentials=creds) with self.assertRaises(Exception): client.get_buckets() + client.close() @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') def test_security_password_without_cacert(self): @@ -79,6 +83,7 @@ def test_security_password_without_cacert(self): val1 = "foobar" key1 = myBucket.new('x', data=val1) key1.store() + client.close() @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') def test_security_cert_authentication(self): @@ -101,6 +106,7 @@ def test_security_cert_authentication(self): with self.assertRaises(Exception): key1.store() myBucket.get('x') + client.close() @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') def test_security_revoked_cert(self): @@ -115,6 +121,7 @@ def test_security_revoked_cert(self): client = self.create_client(credentials=creds) with self.assertRaises(Exception): client.get_buckets() + client.close() @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') def test_security_bad_ca_cert(self): @@ -124,6 +131,7 @@ def test_security_bad_ca_cert(self): client = self.create_client(credentials=creds) with self.assertRaises(Exception): client.get_buckets() + client.close() @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') def test_security_ciphers(self): @@ -136,6 +144,7 @@ def test_security_ciphers(self): key1 = myBucket.new('x', data=val1) key1.store() myBucket.get('x') + client.close() @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') def test_security_bad_ciphers(self): @@ -145,3 +154,4 @@ def test_security_bad_ciphers(self): client = self.create_client(credentials=creds) with self.assertRaises(Exception): client.get_buckets() + client.close() From a4ab50c6c1a0531c0fb4cd52e729a10393177380 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Tue, 27 Oct 2015 09:43:46 -0700 Subject: [PATCH 08/30] Test fixes, decoding TS data fixes, ensure that resources are cleaned up --- commands.py | 11 ++++---- riak/bucket.py | 4 ++- riak/client/multiget.py | 4 ++- riak/tests/base.py | 48 +++++++++++++++++++---------------- riak/tests/test_kv.py | 10 +++----- riak/tests/test_search.py | 9 +++---- riak/tests/test_timeseries.py | 33 ++++++++++++++++++------ riak/tests/yz_setup.py | 11 ++++---- riak/transports/pbc/codec.py | 36 ++++++++++++++++++-------- setup.py | 1 + 10 files changed, 100 insertions(+), 67 deletions(-) diff --git a/commands.py b/commands.py index 0a736284..0f678c02 100644 --- a/commands.py +++ b/commands.py @@ -416,9 +416,9 @@ def _update_riak_conf(self): https_host = self.host + ':' + self.https_port pb_host = self.host + ':' + self.pb_port self._backup_file(self.riak_conf) - f = open(self.riak_conf, 'r', buffering=1) - conf = f.read() - f.close() + conf = None + with open(self.riak_conf, 'r', buffering=1) as f: + conf = f.read() conf = re.sub(r'search\s+=\s+off', r'search = on', conf) conf = re.sub(r'##[ ]+ssl\.', r'ssl.', conf) conf = re.sub(r'ssl.certfile\s+=\s+\S+', @@ -447,9 +447,8 @@ def _update_riak_conf(self): # Older versions of OpenSSL client library need to match on the server conf += 'tls_protocols.tlsv1 = on\n' conf += 'tls_protocols.tlsv1.1 = on\n' - f = open(self.riak_conf, 'w', buffering=1) - f.write(conf) - f.close() + with open(self.riak_conf, 'w', buffering=1) as f: + f.write(conf) def _backup_file(self, name): backup = name + ".bak" diff --git a/riak/bucket.py b/riak/bucket.py index 1d56926f..f6dd3863 100644 --- a/riak/bucket.py +++ b/riak/bucket.py @@ -410,7 +410,9 @@ def new_from_file(self, key, filename): :type filename: string :rtype: :class:`RiakObject ` """ - binary_data = open(filename, "rb").read() + binary_data = None + with open(filename, 'rb') as f: + binary_data = f.read() mimetype, encoding = mimetypes.guess_type(filename) if encoding: binary_data = bytearray(binary_data, encoding) diff --git a/riak/client/multiget.py b/riak/client/multiget.py index a8573cc8..20d02801 100644 --- a/riak/client/multiget.py +++ b/riak/client/multiget.py @@ -209,7 +209,9 @@ def multiget(client, keys, **options): client = RiakClient(protocol='pbc') bkeys = [('default', 'multiget', str(key)) for key in range(10000)] - data = open(__file__).read() + data = None + with open(__file__) as f: + data = f.read() print("Benchmarking multiget:") print(" CPUs: {0}".format(cpu_count())) diff --git a/riak/tests/base.py b/riak/tests/base.py index 0fb4317f..8856f923 100644 --- a/riak/tests/base.py +++ b/riak/tests/base.py @@ -25,25 +25,25 @@ def randname(length=12): out += chr(random.randint(ord('a'), ord('z'))) return out - def create_client(self, host=None, http_port=None, pb_port=None, - protocol=None, credentials=None, - **client_args): - host = host or self.host or HOST - http_port = http_port or self.http_port or HTTP_PORT - pb_port = pb_port or self.pb_port or PB_PORT + @classmethod + def create_client(cls, host=None, http_port=None, pb_port=None, + protocol=None, credentials=None, **client_args): + host = host or HOST + http_port = http_port or HTTP_PORT + pb_port = pb_port or PB_PORT if protocol is None: - if hasattr(self, 'protocol') and (self.protocol is not None): - protocol = self.protocol + if hasattr(cls, 'protocol') and (cls.protocol is not None): + protocol = cls.protocol else: protocol = PROTOCOL - self.protocol = protocol + cls.protocol = protocol credentials = credentials or SECURITY_CREDS - if self.logging_enabled: - self.logger.debug("RiakClient(protocol='%s', host='%s', pb_port='%d', http_port='%d', credentials='%s', client_args='%s')", protocol, host, pb_port, http_port, credentials, client_args) + if hasattr(cls, 'logging_enabled') and cls.logging_enabled: + cls.logger.debug("RiakClient(protocol='%s', host='%s', pb_port='%d', http_port='%d', credentials='%s', client_args='%s')", protocol, host, pb_port, http_port, credentials, client_args) return RiakClient(protocol=protocol, host=host, @@ -51,23 +51,27 @@ def create_client(self, host=None, http_port=None, pb_port=None, credentials=credentials, pb_port=pb_port, **client_args) - def setUp(self): - self.logging_enabled = False + @classmethod + def setUpClass(cls): + cls.logging_enabled = False distutils_debug = os.environ.get('DISTUTILS_DEBUG', '0') if distutils_debug == '1': - self.logging_enabled = True - self.logger = logging.getLogger() - self.logger.level = logging.DEBUG - self.logging_stream_handler = logging.StreamHandler(sys.stdout) - self.logger.addHandler(self.logging_stream_handler) + cls.logging_enabled = True + cls.logger = logging.getLogger() + cls.logger.level = logging.DEBUG + cls.logging_stream_handler = logging.StreamHandler(sys.stdout) + cls.logger.addHandler(cls.logging_stream_handler) + + @classmethod + def tearDownClass(cls): + if hasattr(cls, 'logging_enabled') and cls.logging_enabled: + cls.logger.removeHandler(cls.logging_stream_handler) + cls.logging_enabled = False + def setUp(self): self.bucket_name = self.randname() self.key_name = self.randname() - self.credentials = SECURITY_CREDS self.client = self.create_client() def tearDown(self): self.client.close() - if self.logging_enabled: - self.logger.removeHandler(self.logging_stream_handler) - diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 75bb69f2..592031c5 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -5,9 +5,9 @@ import copy from time import sleep -from riak import ConflictError, RiakClient, RiakBucket, RiakError +from riak import ConflictError, RiakBucket, RiakError from riak.resolver import default_resolver, last_written_resolver -from riak.tests import SKIP_RESOLVE, HOST, PROTOCOL, PB_PORT, HTTP_PORT, SECURITY_CREDS +from riak.tests import SKIP_RESOLVE from riak.tests.base import IntegrationTestBase from riak.tests.comparison import Comparison @@ -35,14 +35,12 @@ testrun_props_bucket = 'propsbucket' def setUpModule(): - c = RiakClient(protocol=PROTOCOL, host=HOST, http_port=HTTP_PORT, - pb_port=PB_PORT, credentials=SECURITY_CREDS) + c = IntegrationTestBase.create_client() c.bucket(testrun_sibs_bucket).allow_mult = True c.close() def tearDownModule(): - c = RiakClient(protocol=PROTOCOL, host=HOST, http_port=HTTP_PORT, - pb_port=PB_PORT, credentials=SECURITY_CREDS) + c = IntegrationTestBase.create_client() c.bucket(testrun_sibs_bucket).clear_properties() c.bucket(testrun_props_bucket).clear_properties() c.close() diff --git a/riak/tests/test_search.py b/riak/tests/test_search.py index 3050de34..73c6cd47 100644 --- a/riak/tests/test_search.py +++ b/riak/tests/test_search.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import print_function import platform -from riak import RiakClient -from riak.tests import SKIP_SEARCH, HOST, PROTOCOL, PB_PORT, HTTP_PORT, SECURITY_CREDS +from riak.tests import SKIP_SEARCH from riak.tests.base import IntegrationTestBase if platform.python_version() < '2.7': @@ -14,16 +13,14 @@ def setUpModule(): if not SKIP_SEARCH and not RUN_YZ: - c = RiakClient(protocol=PROTOCOL, host=HOST, http_port=HTTP_PORT, - pb_port=PB_PORT, credentials=SECURITY_CREDS) + c = IntegrationTestBase.create_client() b = c.bucket(testrun_search_bucket) b.enable_search() c.close() def tearDownModule(): if not SKIP_SEARCH and not RUN_YZ: - c = RiakClient(protocol=PROTOCOL, host=HOST, http_port=HTTP_PORT, - pb_port=PB_PORT, credentials=SECURITY_CREDS) + c = IntegrationTestBase.create_client() b = c.bucket(testrun_search_bucket) b.clear_properties() c.close() diff --git a/riak/tests/test_timeseries.py b/riak/tests/test_timeseries.py index 34aa23b9..b39eb5bd 100644 --- a/riak/tests/test_timeseries.py +++ b/riak/tests/test_timeseries.py @@ -9,9 +9,8 @@ from riak.table import Table from riak.ts_object import TsObject from riak.transports.pbc.codec import RiakPbcCodec -from riak import RiakClient from riak.util import str_to_bytes -from riak.tests import SKIP_TIMESERIES, HOST, PROTOCOL, PB_PORT, HTTP_PORT, SECURITY_CREDS +from riak.tests import SKIP_TIMESERIES from riak.tests.base import IntegrationTestBase if platform.python_version() < '2.7': @@ -179,28 +178,39 @@ def test_decode_data(self): class TimeseriesTests(IntegrationTestBase, unittest.TestCase): @classmethod def setUpClass(cls): + super(TimeseriesTests, cls).setUpClass() cls.now = datetime.datetime.utcfromtimestamp(144379690) fiveMinsAgo = cls.now - fiveMins tenMinsAgo = fiveMinsAgo - fiveMins fifteenMinsAgo = tenMinsAgo - fiveMins twentyMinsAgo = fifteenMinsAgo - fiveMins - client = RiakClient(protocol=PROTOCOL, host=HOST, http_port=HTTP_PORT, - pb_port=PB_PORT, credentials=SECURITY_CREDS) + client = cls.create_client() table = client.table(table_name) + # CREATE TABLE GeoCheckin ( + # geohash varchar not null, + # user varchar not null, + # time timestamp not null, + # weather varchar not null, + # temperature float, + # PRIMARY KEY((quantum(time, 15, m), user), time, user) + # ) rows = [ - [ 'hash1', 'user2', twentyMinsAgo, 'hurricane', None ], + [ 'hash1', 'user2', twentyMinsAgo, 'hurricane', 82.3 ], [ 'hash1', 'user2', fifteenMinsAgo, 'rain', 79.0 ], - [ 'hash1', 'user2', fiveMinsAgo, 'wind', 50.5 ], + [ 'hash1', 'user2', fiveMinsAgo, 'wind', None ], [ 'hash1', 'user2', cls.now, 'snow', 20.1 ] ] ts_obj = table.new(rows) result = ts_obj.store() + if result != True: + raise AssertionError("expected success") + client.close() codec = RiakPbcCodec() cls.nowMsec = codec._unix_time_millis(cls.now) + cls.fiveMinsAgo = fiveMinsAgo cls.tenMinsAgoMsec = codec._unix_time_millis(tenMinsAgo) - client.close() # TODO RTS-367 ts_query test. Ensure that 'None' comes back, somehow def test_query_that_returns_no_data(self): @@ -210,7 +220,14 @@ def test_query_that_returns_no_data(self): self.assertEqual(len(ts_obj.rows), 0) def test_query_that_matches_some_data(self): - query = "select * from {} where time > {} and time < {} and user = 'user2'".format(table_name, self.tenMinsAgoMsec, self.nowMsec); + query = "select * from {} where time > {} and time < {} and user = 'user2'".format(table_name, self.tenMinsAgoMsec, self.nowMsec) ts_obj = self.client.ts_query('GeoCheckin', query) self.assertEqual(len(ts_obj.columns), 5) self.assertEqual(len(ts_obj.rows), 1) + + r0 = ts_obj.rows[0] + self.assertEqual(r0[0], 'hash1') + self.assertEqual(r0[1], 'user2') + self.assertEqual(r0[2], self.fiveMinsAgo) + self.assertEqual(r0[3], 'wind') + self.assertIsNone(r0[4]) diff --git a/riak/tests/yz_setup.py b/riak/tests/yz_setup.py index f7a88efa..80462988 100644 --- a/riak/tests/yz_setup.py +++ b/riak/tests/yz_setup.py @@ -1,12 +1,12 @@ import logging -from riak import RiakClient, RiakError -from riak.tests import RUN_YZ, PROTOCOL, HOST, PB_PORT, HTTP_PORT, SECURITY_CREDS +from riak import RiakError +from riak.tests import RUN_YZ +from riak.tests.base import IntegrationTestBase def yzSetUp(*yzdata): if RUN_YZ: - c = RiakClient(protocol=PROTOCOL, host=HOST, http_port=HTTP_PORT, - pb_port=PB_PORT, credentials=SECURITY_CREDS) + c = IntegrationTestBase.create_client() for yz in yzdata: logging.debug("yzSetUp: %s", yz) c.create_search_index(yz['index'], timeout=30000) @@ -27,8 +27,7 @@ def yzSetUp(*yzdata): def yzTearDown(c, *yzdata): if RUN_YZ: - c = RiakClient(protocol=PROTOCOL, host=HOST, http_port=HTTP_PORT, - pb_port=PB_PORT, credentials=SECURITY_CREDS) + c = IntegrationTestBase.create_client() for yz in yzdata: logging.debug("yzTearDown: %s", yz) if yz['btype'] is not None: diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index d1d7a3af..89fd69de 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -76,7 +76,12 @@ def __init__(self, **unused_args): super(RiakPbcCodec, self).__init__(**unused_args) def _unix_time_millis(self, dt): - return int((dt - epoch).total_seconds() * 1000.0) + td = dt - epoch + try: + return int(dt.total_seconds() * 1000.0) + except AttributeError: + # NB: python 2.6 must use this method + return int(((td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6) * 1000.0) def _datetime_from_unix_time_millis(self, ut): return datetime.datetime.utcfromtimestamp(ut / 1000.0) @@ -710,21 +715,30 @@ def _decode_timeseries_row(self, ts_row, ts_columns): row = [] for i, ts_cell in enumerate(ts_row.cells): ts_col = ts_columns[i] - if ts_col.type == riak_pb.TsColumnType.Value('BINARY'): + logging.debug("ts_cell: '%s', ts_col: '%d'", ts_cell, ts_col.type) + if ts_col.type == riak_pb.TsColumnType.Value('BINARY') and ts_cell.HasField('binary_value'): logging.debug("ts_cell.binary_value: '%s'", ts_cell.binary_value) row.append(ts_cell.binary_value) - elif ts_col.type == riak_pb.TsColumnType.Value('INTEGER'): + elif ts_col.type == riak_pb.TsColumnType.Value('INTEGER') and ts_cell.HasField('integer_value'): logging.debug("ts_cell.integer_value: '%s'", ts_cell.integer_value) row.append(ts_cell.integer_value) - elif ts_col.type == riak_pb.TsColumnType.Value('FLOAT'): - logging.debug("ts_cell.double_value: '%s'", ts_cell.double_value) - row.append(ts_cell.double_value) + elif ts_col.type == riak_pb.TsColumnType.Value('FLOAT') and ts_cell.HasField('double_value'): + value = None + if ts_cell.HasField('double_value'): + value = ts_cell.double_value + elif ts_cell.HasField('float_value'): + value = ts_cell.float_value + logging.debug("ts_cell double/float value: '%d'", value) + row.append(value) elif ts_col.type == riak_pb.TsColumnType.Value('TIMESTAMP'): - dt = self._datetime_from_unix_time_millis(ts_cell.timestamp_value) - logging.debug("ts_cell.timestamp_value: '%s', datetime: '%s'", - ts_cell.timestamp_value, dt) + dt = None + if ts_cell.HasField('timestamp_value'): + dt = self._datetime_from_unix_time_millis(ts_cell.timestamp_value) + elif ts_cell.HasField('integer_value'): + dt = self._datetime_from_unix_time_millis(ts_cell.integer_value) + logging.debug("ts_cell datetime: '%s'", dt) row.append(dt) - elif ts_col.type == riak_pb.TsColumnType.Value('BOOLEAN'): + elif ts_col.type == riak_pb.TsColumnType.Value('BOOLEAN') and ts_cell.HasField('boolean_value'): logging.debug("ts_cell.boolean_value: '%s'", ts_cell.boolean_value) row.append(ts_cell.boolean_value) elif ts_col.type == riak_pb.TsColumnType.Value('SET'): @@ -734,7 +748,7 @@ def _decode_timeseries_row(self, ts_row, ts_columns): sj = bytes_to_str(sv) s.append(json.loads(sj)) row.append(s) - elif ts_col.type == riak_pb.TsColumnType.Value('MAP'): + elif ts_col.type == riak_pb.TsColumnType.Value('MAP') and ts_cell.HasField('map_value'): logging.debug("ts_cell.map_value: '%s'", ts_cell.map_value) mj = bytes_to_str(ts_cell.map_value) row.append(json.loads(mj)) diff --git a/setup.py b/setup.py index 3e2b84d1..d5ba1bd9 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ #!/usr/bin/env python import os import sys +from multiprocessing import util from setuptools import setup, find_packages from version import get_version from commands import preconfigure, configure, create_bucket_types, \ From 867ff6310e6422082ae8cd7460c143840e823f94 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Tue, 27 Oct 2015 09:47:21 -0700 Subject: [PATCH 09/30] Remove two TODOs --- riak/tests/test_timeseries.py | 1 - riak/transports/http/__init__.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/riak/tests/test_timeseries.py b/riak/tests/test_timeseries.py index b39eb5bd..995eeafa 100644 --- a/riak/tests/test_timeseries.py +++ b/riak/tests/test_timeseries.py @@ -212,7 +212,6 @@ def setUpClass(cls): cls.fiveMinsAgo = fiveMinsAgo cls.tenMinsAgoMsec = codec._unix_time_millis(tenMinsAgo) - # TODO RTS-367 ts_query test. Ensure that 'None' comes back, somehow def test_query_that_returns_no_data(self): query = "select * from {} where time > 0 and time < 10 and user = 'user1'".format(table_name) ts_obj = self.client.ts_query('GeoCheckin', query) diff --git a/riak/transports/http/__init__.py b/riak/transports/http/__init__.py index c4a19a96..4acb06fd 100644 --- a/riak/transports/http/__init__.py +++ b/riak/transports/http/__init__.py @@ -87,7 +87,7 @@ def __init__(self, :type timeout: int """ if PY2: - # TODO LRB RTS-367 it appears that pkey_file / cert_file are never set + # NB: it appears that pkey_file / cert_file are never set # in riak/transports/http/connection.py#_connect() method pkf = pkey_file if pkf is None and credentials is not None: From c268e6aa41e489ba6085c94b3cca4b36b2387838 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Tue, 27 Oct 2015 15:51:33 -0700 Subject: [PATCH 10/30] Add TsGetReq / TsGetResp support and tests --- docs/client.rst | 1 + riak/client/operations.py | 21 +++++++ riak/table.py | 12 ++++ riak/tests/test_timeseries.py | 53 ++++++++++++---- riak/transports/pbc/codec.py | 102 ++++++++++++++++++------------- riak/transports/pbc/transport.py | 18 +++++- 6 files changed, 149 insertions(+), 58 deletions(-) diff --git a/docs/client.rst b/docs/client.rst index a2a3b135..e932d206 100644 --- a/docs/client.rst +++ b/docs/client.rst @@ -127,6 +127,7 @@ Key-level Operations Timeseries Operations -------------------- +.. automethod:: RiakClient.ts_get .. automethod:: RiakClient.ts_put .. automethod:: RiakClient.ts_query diff --git a/riak/client/operations.py b/riak/client/operations.py index f1881fcf..573e41c8 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -536,6 +536,27 @@ def put(self, transport, robj, w=None, dw=None, pw=None, return_body=None, if_none_match=if_none_match, timeout=timeout) + @retryable + def ts_get(self, transport, table, key): + """ + ts_get(table, key) + + Retrieve timeseries value by key + + .. note:: This request is automatically retried :attr:`retries` + times if it fails due to network error. + + :param table: The timeseries table. + :type table: string or :class:`Table ` + :param key: The timeseries value's key. + :type key: list or dict + :rtype: :class:`TsObject ` + """ + t = table + if isinstance(t, str): + t = Table(self, table) + return transport.ts_get(t, key) + @retryable def ts_put(self, transport, tsobj): """ diff --git a/riak/table.py b/riak/table.py index 10da507c..c6f47f12 100644 --- a/riak/table.py +++ b/riak/table.py @@ -48,6 +48,18 @@ def new(self, rows, columns=None): return TsObject(self._client, self, rows, columns) + def get(self, table, key): + """ + Gets a value from a timeseries table. + + :param table: The timeseries table. + :type table: string or :class:`Table ` + :param key: The timeseries value's key. + :type key: list or dict + :rtype: :class:`TsObject ` + """ + return self.client.ts_get(self, table, key) + def query(self, query, interpolations=None): """ Queries a timeseries table. diff --git a/riak/tests/test_timeseries.py b/riak/tests/test_timeseries.py index 995eeafa..a2e6fc87 100644 --- a/riak/tests/test_timeseries.py +++ b/riak/tests/test_timeseries.py @@ -9,7 +9,7 @@ from riak.table import Table from riak.ts_object import TsObject from riak.transports.pbc.codec import RiakPbcCodec -from riak.util import str_to_bytes +from riak.util import str_to_bytes, bytes_to_str from riak.tests import SKIP_TIMESERIES from riak.tests.base import IntegrationTestBase @@ -48,12 +48,26 @@ def setUp(self): ] self.table = Table(None, 'test-table') - def test_encode_data(self): + def test_encode_data_for_get(self): + key = { + 'user' : 'user2', + 'time' : ts0 + } + ts_get_req = riak_pb.TsGetReq() + self.c._encode_timeseries_get(self.table, key, ts_get_req) + + self.assertEqual(self.table.name, bytes_to_str(ts_get_req.table)) + self.assertEqual(len(key.values()), len(ts_get_req.key)) + self.assertEqual('user2', bytes_to_str(ts_get_req.key[0].binary_value)) + self.assertEqual(self.ts0ms, ts_get_req.key[1].timestamp_value) + + def test_encode_data_for_put(self): tsobj = TsObject(None, self.table, self.rows, None) ts_put_req = riak_pb.TsPutReq() - self.c._encode_timeseries(tsobj, ts_put_req) + self.c._encode_timeseries_put(tsobj, ts_put_req) # NB: expected, actual + self.assertEqual(self.table.name, bytes_to_str(ts_put_req.table)) self.assertEqual(len(self.rows), len(ts_put_req.rows)) r0 = ts_put_req.rows[0] @@ -74,7 +88,7 @@ def test_encode_data(self): self.assertEqual(r1.cells[5].set_value, sj) self.assertEqual(r1.cells[6].map_value, mj) - def test_decode_data(self): + def test_decode_data_from_query(self): tqr = riak_pb.TsQueryResp() c0 = tqr.columns.add() @@ -212,6 +226,16 @@ def setUpClass(cls): cls.fiveMinsAgo = fiveMinsAgo cls.tenMinsAgoMsec = codec._unix_time_millis(tenMinsAgo) + def validate_data(self, ts_obj): + self.assertEqual(len(ts_obj.columns), 5) + self.assertEqual(len(ts_obj.rows), 1) + row = ts_obj.rows[0] + self.assertEqual(row[0], 'hash1') + self.assertEqual(row[1], 'user2') + self.assertEqual(row[2], self.fiveMinsAgo) + self.assertEqual(row[3], 'wind') + self.assertIsNone(row[4]) + def test_query_that_returns_no_data(self): query = "select * from {} where time > 0 and time < 10 and user = 'user1'".format(table_name) ts_obj = self.client.ts_query('GeoCheckin', query) @@ -221,12 +245,17 @@ def test_query_that_returns_no_data(self): def test_query_that_matches_some_data(self): query = "select * from {} where time > {} and time < {} and user = 'user2'".format(table_name, self.tenMinsAgoMsec, self.nowMsec) ts_obj = self.client.ts_query('GeoCheckin', query) - self.assertEqual(len(ts_obj.columns), 5) - self.assertEqual(len(ts_obj.rows), 1) + self.validate_data(ts_obj) + + def test_get_single_value_using_dict(self): + key = { + 'user' : 'user2', + 'time' : self.fiveMinsAgo + } + ts_obj = self.client.ts_get('GeoCheckin', key) + self.validate_data(ts_obj) - r0 = ts_obj.rows[0] - self.assertEqual(r0[0], 'hash1') - self.assertEqual(r0[1], 'user2') - self.assertEqual(r0[2], self.fiveMinsAgo) - self.assertEqual(r0[3], 'wind') - self.assertIsNone(r0[4]) + def test_get_single_value_using_array(self): + key = [ self.fiveMinsAgo, 'user2' ] + ts_obj = self.client.ts_get('GeoCheckin', key) + self.validate_data(ts_obj) diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index 89fd69de..41f03cbb 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -623,7 +623,55 @@ def _encode_map_update(self, dtype, msg, op): else: msg.flag_op = riak_pb.MapUpdate.DISABLE - def _encode_timeseries(self, tsobj, ts_put_req): + def _encode_to_ts_cell(self, cell, ts_cell): + if cell is not None: + if isinstance(cell, bytes) or isinstance(cell, bytearray): + logging.debug("cell -> binary_value: '%s'", cell) + ts_cell.binary_value = cell + elif isinstance(cell, datetime.datetime): + ts_cell.timestamp_value = self._unix_time_millis(cell) + logging.debug("cell -> timestamp: '%s', timestamp_value '%d'", + cell, ts_cell.timestamp_value) + elif isinstance(cell, list): + for c in cell: + j = json.dumps(c) + logging.debug("cell -> set_value: '%s'", j) + ts_cell.set_value.append(str_to_bytes(j)) + elif isinstance(cell, bool): + logging.debug("cell -> boolean: '%s'", cell) + ts_cell.boolean_value = cell + elif isinstance(cell, str): + logging.debug("cell -> str: '%s'", cell) + ts_cell.binary_value = str_to_bytes(cell) + elif isinstance(cell, int) or isinstance(cell, long): + logging.debug("cell -> int/long: '%s'", cell) + ts_cell.integer_value = cell + elif isinstance(cell, float): + logging.debug("cell -> float: '%s'", cell) + ts_cell.double_value = cell + elif isinstance(cell, dict): + logging.debug("cell -> dict: '%s'", cell) + j = json.dumps(cell) + ts_cell.map_value = str_to_bytes(j) + else: + t = type(cell) + raise RiakError("can't serialize type '{}', value '{}'".format(t, cell)) + + def _encode_timeseries_get(self, table, key, req): + key_vals = None + if isinstance(key, list): + key_vals = key + elif isinstance(key, dict): + key_vals = key.values() + else: + raise ValueError("key must be a list or dict") + + req.table = str_to_bytes(table.name) + for cell in key_vals: + ts_cell = req.key.add() + self._encode_to_ts_cell(cell, ts_cell) + + def _encode_timeseries_put(self, tsobj, ts_put_req): """ Fills an TsPutReq message with the appropriate data and metadata from a TsObject. @@ -634,73 +682,41 @@ def _encode_timeseries(self, tsobj, ts_put_req): :type ts_put_req: riak_pb.TsPutReq """ ts_put_req.table = str_to_bytes(tsobj.table.name) + if tsobj.columns: raise NotImplementedError("columns are not implemented yet") + if tsobj.rows and isinstance(tsobj.rows, list): for row in tsobj.rows: tsr = ts_put_req.rows.add() # NB: type riak_pb.TsRow if not isinstance(row, list): - raise RiakError("TsObject row must be a list of values") + raise ValueError("TsObject row must be a list of values") for cell in row: tsc = tsr.cells.add() # NB: type riak_pb.TsCell - if cell is not None: - if isinstance(cell, bytes) or isinstance(cell, bytearray): - logging.debug("cell -> binary_value: '%s'", cell) - tsc.binary_value = cell - elif isinstance(cell, datetime.datetime): - tsc.timestamp_value = self._unix_time_millis(cell) - logging.debug("cell -> timestamp: '%s', timestamp_value '%d'", - cell, tsc.timestamp_value) - elif isinstance(cell, list): - for c in cell: - j = json.dumps(c) - logging.debug("cell -> set_value: '%s'", j) - tsc.set_value.append(str_to_bytes(j)) - elif isinstance(cell, bool): - logging.debug("cell -> boolean: '%s'", cell) - tsc.boolean_value = cell - elif isinstance(cell, str): - logging.debug("cell -> str: '%s'", cell) - tsc.binary_value = str_to_bytes(cell) - elif isinstance(cell, int) or isinstance(cell, long): - logging.debug("cell -> int/long: '%s'", cell) - tsc.integer_value = cell - elif isinstance(cell, float): - logging.debug("cell -> float: '%s'", cell) - tsc.double_value = cell - elif isinstance(cell, dict): - logging.debug("cell -> dict: '%s'", cell) - j = json.dumps(cell) - tsc.map_value = str_to_bytes(j) - else: - t = type(cell) - raise RiakError("can't serialize type '{}', value '{}'".format(t, cell)) + self._encode_to_ts_cell(cell, tsc) else: raise RiakError("TsObject requires a list of rows") - def _decode_timeseries(self, ts_query_rsp, tsobj): + def _decode_timeseries(self, ts_rsp, tsobj): """ Fills an TsObject with the appropriate data and metadata from a TsQueryResp. - :param ts_query_rsp: the protobuf message from which to process data - :type ts_query_rsp: riak_pb.TsQueryRsp + :param ts_rsp: the protobuf message from which to process data + :type ts_rsp: riak_pb.TsQueryRsp or riak_pb.TsGetResp :param tsobj: a TsObject :type tsobj: TsObject """ - if not isinstance(ts_query_rsp, riak_pb.TsQueryResp): - raise RiakError("expected riak_pb.TsQueryResp") - if tsobj.columns is not None: - for ts_col in ts_query_rsp.columns: + for ts_col in ts_rsp.columns: col_name = bytes_to_str(ts_col.name) col_type = ts_col.type col = (col_name, col_type) logging.debug("column: '%s'", col) tsobj.columns.append(col) - for ts_row in ts_query_rsp.rows: - tsobj.rows.append(self._decode_timeseries_row(ts_row, ts_query_rsp.columns)) + for ts_row in ts_rsp.rows: + tsobj.rows.append(self._decode_timeseries_row(ts_row, ts_rsp.columns)) def _decode_timeseries_row(self, ts_row, ts_columns): """ diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 8a81b26c..6bbc0105 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -63,7 +63,9 @@ MSG_CODE_TS_PUT_REQ, MSG_CODE_TS_PUT_RESP, MSG_CODE_TS_QUERY_REQ, - MSG_CODE_TS_QUERY_RESP + MSG_CODE_TS_QUERY_RESP, + MSG_CODE_TS_GET_REQ, + MSG_CODE_TS_GET_RESP ) @@ -217,10 +219,20 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, return robj + def ts_get(self, table, key): + req = riak_pb.TsGetReq() + self._encode_timeseries_get(table, key, req) + + msg_code, ts_get_resp = self._request(MSG_CODE_TS_GET_REQ, req, + MSG_CODE_TS_GET_RESP) + + tsobj = TsObject(self._client, table, [], None) + self._decode_timeseries(ts_get_resp, tsobj) + return tsobj + def ts_put(self, tsobj): req = riak_pb.TsPutReq() - - self._encode_timeseries(tsobj, req) + self._encode_timeseries_put(tsobj, req) msg_code, resp = self._request(MSG_CODE_TS_PUT_REQ, req, MSG_CODE_TS_PUT_RESP) From aece9f32f796878c233200cdff0efa2e541276ef Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Tue, 27 Oct 2015 16:09:27 -0700 Subject: [PATCH 11/30] Add TsDelReq / TsDelResp support and tests --- docs/client.rst | 1 + riak/client/operations.py | 21 ++++++++++++++++++ riak/table.py | 12 ++++++++++ riak/tests/test_timeseries.py | 38 +++++++++++++++++++++++--------- riak/transports/pbc/codec.py | 2 +- riak/transports/pbc/transport.py | 22 ++++++++++++++---- 6 files changed, 81 insertions(+), 15 deletions(-) diff --git a/docs/client.rst b/docs/client.rst index e932d206..ae2f8e54 100644 --- a/docs/client.rst +++ b/docs/client.rst @@ -129,6 +129,7 @@ Timeseries Operations .. automethod:: RiakClient.ts_get .. automethod:: RiakClient.ts_put +.. automethod:: RiakClient.ts_delete .. automethod:: RiakClient.ts_query ---------------- diff --git a/riak/client/operations.py b/riak/client/operations.py index 573e41c8..4f2d20a7 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -573,6 +573,27 @@ def ts_put(self, transport, tsobj): """ return transport.ts_put(tsobj) + @retryable + def ts_delete(self, transport, table, key): + """ + ts_delete(table, key) + + Delete timeseries value by key + + .. note:: This request is automatically retried :attr:`retries` + times if it fails due to network error. + + :param table: The timeseries table. + :type table: string or :class:`Table ` + :param key: The timeseries value's key. + :type key: list or dict + :rtype: boolean + """ + t = table + if isinstance(t, str): + t = Table(self, table) + return transport.ts_delete(t, key) + @retryable def ts_query(self, transport, table, query, interpolations=None): """ diff --git a/riak/table.py b/riak/table.py index c6f47f12..6635cd0a 100644 --- a/riak/table.py +++ b/riak/table.py @@ -60,6 +60,18 @@ def get(self, table, key): """ return self.client.ts_get(self, table, key) + def delete(self, table, key): + """ + Deletes a value from a timeseries table. + + :param table: The timeseries table. + :type table: string or :class:`Table ` + :param key: The timeseries value's key. + :type key: list or dict + :rtype: boolean + """ + return self.client.ts_delete(self, table, key) + def query(self, query, interpolations=None): """ Queries a timeseries table. diff --git a/riak/tests/test_timeseries.py b/riak/tests/test_timeseries.py index a2e6fc87..971d2355 100644 --- a/riak/tests/test_timeseries.py +++ b/riak/tests/test_timeseries.py @@ -46,20 +46,27 @@ def setUp(self): [ bd0, 0, 1.2, ts0, True, s, m ], [ bd1, 3, 4.5, ts1, False, s, m ] ] - self.table = Table(None, 'test-table') - - def test_encode_data_for_get(self): - key = { + self.test_key = { 'user' : 'user2', 'time' : ts0 } - ts_get_req = riak_pb.TsGetReq() - self.c._encode_timeseries_get(self.table, key, ts_get_req) + self.table = Table(None, 'test-table') + + def validate_keyreq(self, req): + self.assertEqual(self.table.name, bytes_to_str(req.table)) + self.assertEqual(len(self.test_key.values()), len(req.key)) + self.assertEqual('user2', bytes_to_str(req.key[0].binary_value)) + self.assertEqual(self.ts0ms, req.key[1].timestamp_value) - self.assertEqual(self.table.name, bytes_to_str(ts_get_req.table)) - self.assertEqual(len(key.values()), len(ts_get_req.key)) - self.assertEqual('user2', bytes_to_str(ts_get_req.key[0].binary_value)) - self.assertEqual(self.ts0ms, ts_get_req.key[1].timestamp_value) + def test_encode_data_for_get(self): + req = riak_pb.TsGetReq() + self.c._encode_timeseries_keyreq(self.table, self.test_key, req) + self.validate_keyreq(req) + + def test_encode_data_for_delete(self): + req = riak_pb.TsDelReq() + self.c._encode_timeseries_keyreq(self.table, self.test_key, req) + self.validate_keyreq(req) def test_encode_data_for_put(self): tsobj = TsObject(None, self.table, self.rows, None) @@ -224,6 +231,7 @@ def setUpClass(cls): codec = RiakPbcCodec() cls.nowMsec = codec._unix_time_millis(cls.now) cls.fiveMinsAgo = fiveMinsAgo + cls.twentyMinsAgo = twentyMinsAgo cls.tenMinsAgoMsec = codec._unix_time_millis(tenMinsAgo) def validate_data(self, ts_obj): @@ -259,3 +267,13 @@ def test_get_single_value_using_array(self): key = [ self.fiveMinsAgo, 'user2' ] ts_obj = self.client.ts_get('GeoCheckin', key) self.validate_data(ts_obj) + + def test_delete_single_value_using_dict(self): + key = { + 'user' : 'user2', + 'time' : self.twentyMinsAgo + } + rslt = self.client.ts_delete('GeoCheckin', key) + self.assertTrue(rslt) + ts_obj = self.client.ts_get('GeoCheckin', key) + self.assertIsNone(ts_obj) diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index 41f03cbb..be85719e 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -657,7 +657,7 @@ def _encode_to_ts_cell(self, cell, ts_cell): t = type(cell) raise RiakError("can't serialize type '{}', value '{}'".format(t, cell)) - def _encode_timeseries_get(self, table, key, req): + def _encode_timeseries_keyreq(self, table, key, req): key_vals = None if isinstance(key, list): key_vals = key diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 6bbc0105..4d642fce 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -65,7 +65,9 @@ MSG_CODE_TS_QUERY_REQ, MSG_CODE_TS_QUERY_RESP, MSG_CODE_TS_GET_REQ, - MSG_CODE_TS_GET_RESP + MSG_CODE_TS_GET_RESP, + MSG_CODE_TS_DEL_REQ, + MSG_CODE_TS_DEL_RESP ) @@ -221,10 +223,10 @@ def put(self, robj, w=None, dw=None, pw=None, return_body=True, def ts_get(self, table, key): req = riak_pb.TsGetReq() - self._encode_timeseries_get(table, key, req) + self._encode_timeseries_keyreq(table, key, req) msg_code, ts_get_resp = self._request(MSG_CODE_TS_GET_REQ, req, - MSG_CODE_TS_GET_RESP) + MSG_CODE_TS_GET_RESP) tsobj = TsObject(self._client, table, [], None) self._decode_timeseries(ts_get_resp, tsobj) @@ -239,7 +241,19 @@ def ts_put(self, tsobj): if resp is not None: return True - elif not robj.key: + else: + raise RiakError("missing response object") + + def ts_delete(self, table, key): + req = riak_pb.TsDelReq() + self._encode_timeseries_keyreq(table, key, req) + + msg_code, ts_del_resp = self._request(MSG_CODE_TS_DEL_REQ, req, + MSG_CODE_TS_DEL_RESP) + + if ts_del_resp is not None: + return True + else: raise RiakError("missing response object") def ts_query(self, table, query, interpolations=None): From 429b5fdfb756d65253dce5ca20faa65a63440eb6 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Tue, 27 Oct 2015 16:38:53 -0700 Subject: [PATCH 12/30] Remove support for sets and maps in TS data --- riak/tests/test_timeseries.py | 42 ++--------------------------------- riak/transports/pbc/codec.py | 21 ------------------ 2 files changed, 2 insertions(+), 61 deletions(-) diff --git a/riak/tests/test_timeseries.py b/riak/tests/test_timeseries.py index 971d2355..dbc1674d 100644 --- a/riak/tests/test_timeseries.py +++ b/riak/tests/test_timeseries.py @@ -27,24 +27,14 @@ ts0 = datetime.datetime(2015, 1, 1, 12, 0, 0) ts1 = ts0 + fiveMins -s = [ 'foo', 'bar', 'baz' ] -m = { - 'foo': 'foo', - 'bar': 'bar', - 'baz': 'baz', - 'set': s -} -sj = ['"foo"', '"bar"', '"baz"'] -mj = '{"baz": "baz", "set": ["foo", "bar", "baz"], "foo": "foo", "bar": "bar"}' - class TimeseriesUnitTests(unittest.TestCase): def setUp(self): self.c = RiakPbcCodec() self.ts0ms = self.c._unix_time_millis(ts0) self.ts1ms = self.c._unix_time_millis(ts1) self.rows = [ - [ bd0, 0, 1.2, ts0, True, s, m ], - [ bd1, 3, 4.5, ts1, False, s, m ] + [ bd0, 0, 1.2, ts0, True ], + [ bd1, 3, 4.5, ts1, False ] ] self.test_key = { 'user' : 'user2', @@ -83,8 +73,6 @@ def test_encode_data_for_put(self): self.assertEqual(r0.cells[2].double_value, self.rows[0][2]) self.assertEqual(r0.cells[3].timestamp_value, self.ts0ms) self.assertEqual(r0.cells[4].boolean_value, self.rows[0][4]) - self.assertEqual(r0.cells[5].set_value, sj) - self.assertEqual(r0.cells[6].map_value, mj) r1 = ts_put_req.rows[1] self.assertEqual(r1.cells[0].binary_value, self.rows[1][0]) @@ -92,8 +80,6 @@ def test_encode_data_for_put(self): self.assertEqual(r1.cells[2].double_value, self.rows[1][2]) self.assertEqual(r1.cells[3].timestamp_value, self.ts1ms) self.assertEqual(r1.cells[4].boolean_value, self.rows[1][4]) - self.assertEqual(r1.cells[5].set_value, sj) - self.assertEqual(r1.cells[6].map_value, mj) def test_decode_data_from_query(self): tqr = riak_pb.TsQueryResp() @@ -113,12 +99,6 @@ def test_decode_data_from_query(self): c4 = tqr.columns.add() c4.name = str_to_bytes('col_boolean') c4.type = riak_pb.TsColumnType.Value('BOOLEAN') - c5 = tqr.columns.add() - c5.name = str_to_bytes('col_set') - c5.type = riak_pb.TsColumnType.Value('SET') - c6 = tqr.columns.add() - c6.name = str_to_bytes('col_map') - c6.type = riak_pb.TsColumnType.Value('MAP') r0 = tqr.rows.add() r0c0 = r0.cells.add() @@ -131,11 +111,6 @@ def test_decode_data_from_query(self): r0c3.timestamp_value = self.ts0ms r0c4 = r0.cells.add() r0c4.boolean_value = self.rows[0][4] - r0c5 = r0.cells.add() - for j in sj: - r0c5.set_value.append(j) - r0c6 = r0.cells.add() - r0c6.map_value = str_to_bytes(mj) r1 = tqr.rows.add() r1c0 = r1.cells.add() @@ -148,11 +123,6 @@ def test_decode_data_from_query(self): r1c3.timestamp_value = self.ts1ms r1c4 = r1.cells.add() r1c4.boolean_value = self.rows[1][4] - r1c5 = r1.cells.add() - for j in sj: - r1c5.set_value.append(j) - r1c6 = r1.cells.add() - r1c6.map_value = str_to_bytes(mj) tsobj = TsObject(None, self.table, [], []) c = RiakPbcCodec() @@ -172,10 +142,6 @@ def test_decode_data_from_query(self): self.assertEqual(c[3][1], riak_pb.TsColumnType.Value('TIMESTAMP')) self.assertEqual(c[4][0], 'col_boolean') self.assertEqual(c[4][1], riak_pb.TsColumnType.Value('BOOLEAN')) - self.assertEqual(c[5][0], 'col_set') - self.assertEqual(c[5][1], riak_pb.TsColumnType.Value('SET')) - self.assertEqual(c[6][0], 'col_map') - self.assertEqual(c[6][1], riak_pb.TsColumnType.Value('MAP')) r0 = tsobj.rows[0] self.assertEqual(r0[0], self.rows[0][0]) @@ -183,8 +149,6 @@ def test_decode_data_from_query(self): self.assertEqual(r0[2], self.rows[0][2]) self.assertEqual(r0[3], ts0) self.assertEqual(r0[4], self.rows[0][4]) - self.assertEqual(r0[5], s) - self.assertEqual(r0[6], m) r1 = tsobj.rows[1] self.assertEqual(r1[0], self.rows[1][0]) @@ -192,8 +156,6 @@ def test_decode_data_from_query(self): self.assertEqual(r1[2], self.rows[1][2]) self.assertEqual(r1[3], ts1) self.assertEqual(r1[4], self.rows[1][4]) - self.assertEqual(r1[5], s) - self.assertEqual(r1[6], m) @unittest.skipIf(SKIP_TIMESERIES == 1, "skip requested for timeseries tests") class TimeseriesTests(IntegrationTestBase, unittest.TestCase): diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index be85719e..f2331f31 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -1,5 +1,4 @@ import datetime -import json import logging import riak_pb @@ -632,11 +631,6 @@ def _encode_to_ts_cell(self, cell, ts_cell): ts_cell.timestamp_value = self._unix_time_millis(cell) logging.debug("cell -> timestamp: '%s', timestamp_value '%d'", cell, ts_cell.timestamp_value) - elif isinstance(cell, list): - for c in cell: - j = json.dumps(c) - logging.debug("cell -> set_value: '%s'", j) - ts_cell.set_value.append(str_to_bytes(j)) elif isinstance(cell, bool): logging.debug("cell -> boolean: '%s'", cell) ts_cell.boolean_value = cell @@ -649,10 +643,6 @@ def _encode_to_ts_cell(self, cell, ts_cell): elif isinstance(cell, float): logging.debug("cell -> float: '%s'", cell) ts_cell.double_value = cell - elif isinstance(cell, dict): - logging.debug("cell -> dict: '%s'", cell) - j = json.dumps(cell) - ts_cell.map_value = str_to_bytes(j) else: t = type(cell) raise RiakError("can't serialize type '{}', value '{}'".format(t, cell)) @@ -757,17 +747,6 @@ def _decode_timeseries_row(self, ts_row, ts_columns): elif ts_col.type == riak_pb.TsColumnType.Value('BOOLEAN') and ts_cell.HasField('boolean_value'): logging.debug("ts_cell.boolean_value: '%s'", ts_cell.boolean_value) row.append(ts_cell.boolean_value) - elif ts_col.type == riak_pb.TsColumnType.Value('SET'): - logging.debug("ts_cell.set_value: '%s'", ts_cell.set_value) - s = [] - for sv in ts_cell.set_value: - sj = bytes_to_str(sv) - s.append(json.loads(sj)) - row.append(s) - elif ts_col.type == riak_pb.TsColumnType.Value('MAP') and ts_cell.HasField('map_value'): - logging.debug("ts_cell.map_value: '%s'", ts_cell.map_value) - mj = bytes_to_str(ts_cell.map_value) - row.append(json.loads(mj)) else: row.append(None) return row From 9cba4512c76d962d4fdbf2983e52a2daa569e9ef Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Wed, 28 Oct 2015 19:15:23 +0000 Subject: [PATCH 13/30] - Rename all SKIP_ variables to RUN_ to be consistent - Clean up all PEP8 and pyflakes warnings --- README.rst | 24 ++++++++++++----- buildbot/Makefile | 8 +++--- commands.py | 21 ++++++++++++--- riak/table.py | 4 ++- riak/tests/__init__.py | 18 +++++++------ riak/tests/base.py | 11 +++++++- riak/tests/test_2i.py | 36 ++++++++++++------------- riak/tests/test_btypes.py | 12 ++------- riak/tests/test_client.py | 5 ++-- riak/tests/test_datatypes.py | 23 ++++------------ riak/tests/test_kv.py | 10 +++---- riak/tests/test_mapreduce.py | 8 ++++-- riak/tests/test_pool.py | 4 +-- riak/tests/test_search.py | 25 +++++++---------- riak/tests/test_security.py | 37 ++++++++++++------------- riak/tests/test_timeseries.py | 44 ++++++++++++++++-------------- riak/tests/test_yokozuna.py | 23 +++++++--------- riak/tests/yz_setup.py | 2 ++ riak/transports/http/__init__.py | 1 - riak/transports/pbc/codec.py | 46 +++++++++++++++++++++----------- riak/ts_object.py | 1 + setup.py | 1 - 22 files changed, 197 insertions(+), 167 deletions(-) diff --git a/README.rst b/README.rst index 94b429b6..b2cb0b65 100644 --- a/README.rst +++ b/README.rst @@ -139,6 +139,8 @@ If your Riak server isn't running on localhost or you have built a Riak devrel from source, use the environment variables ``RIAK_TEST_HOST``, ``RIAK_TEST_HTTP_PORT`` and ``RIAK_TEST_PB_PORT`` to specify where to find the Riak server. +``RIAK_TEST_PROTOCOL`` to specify which protocol to test. Can be +either ``pbc`` or ``http``. Some of the connection tests need port numbers that are NOT in use. If ports 1023 and 1022 are in use on your test system, set the @@ -150,7 +152,7 @@ Testing Search If you don't have `Riak Search `_ enabled, you -can set the ``SKIP_SEARCH`` environment variable to 1 skip those +can set the ``RUN_SEARCH`` environment variable to 0 skip those tests. If you don't have `Search 2.0 `_ @@ -176,10 +178,18 @@ You may alternately add these lines to `setup.cfg` [create_bucket_types] riak-admin=/Users/sean/dev/riak/rel/riak/bin/riak-admin -To skip the bucket-type tests, set the ``SKIP_BTYPES`` environment -variable to ``1``. +To skip the bucket-type tests, set the ``RUN_BTYPES`` environment +variable to ``0``. + +Testing Data Types (Riak 2+) +---------------------------- + +To test data types, you must set up bucket types (see above.) -Testing Timeseries (Riak 2+) +To skip the data type tests, set the ``RUN_DATATYPES`` environment +variable to ``0``. + +Testing Timeseries (Riak 2.1+) ------------------------------ To test timeseries data, you must run the ``setup_timeseries`` command, @@ -198,15 +208,15 @@ You may alternately add these lines to `setup.cfg` [setup_timeseries] riak-admin=/Users/sean/dev/riak/rel/riak/bin/riak-admin -To enable the timeseries tests, set the ``SKIP_TIMESERIES`` environment -variable to ``0``. +To enable the timeseries tests, set the ``RUN_TIMESERIES`` environment +variable to ``1``. Testing Secondary Indexes ------------------------- To test `Secondary Indexes `_, -the ``SKIP_INDEX`` environment variable must be set to 0 (or 1 to skip them.) +the ``RUN_INDEXES`` environment variable must be set to 1 (or 0 to skip them.) Testing Security (Riak 2+) -------------------------- diff --git a/buildbot/Makefile b/buildbot/Makefile index 1209ac84..890e3984 100644 --- a/buildbot/Makefile +++ b/buildbot/Makefile @@ -26,14 +26,14 @@ test: setup test_normal test_security test_normal: @echo "Testing Riak Python Client (without security)" @../setup.py disable_security --riak-admin=${RIAK_ADMIN} - @RIAK_TEST_PROTOCOL='pbc' RUN_YZ=1 SKIP_DATATYPES=0 SKIP_INDEXES=0 ./tox_runner.sh .. - @RIAK_TEST_PROTOCOL='http' RUN_YZ=1 SKIP_DATATYPES=0 SKIP_INDEXES=0 ./tox_runner.sh .. + @RIAK_TEST_PROTOCOL='pbc' RUN_YZ=1 RUN_DATATYPES=1 RUN_INDEXES=1 ./tox_runner.sh .. + @RIAK_TEST_PROTOCOL='http' RUN_YZ=1 RUN_DATATYPES=1 RUN_INDEXES=1 ./tox_runner.sh .. test_security: @echo "Testing Riak Python Client (with security)" @../setup.py enable_security --riak-admin=${RIAK_ADMIN} - @RIAK_TEST_PROTOCOL='pbc' RUN_YZ=1 SKIP_INDEXES=0 RUN_SECURITY=1 SKIP_POOL=1 SKIP_RESOLVE=1 ./tox_runner.sh .. - @RIAK_TEST_PROTOCOL='http' RUN_YZ=1 SKIP_INDEXES=0 RUN_SECURITY=1 SKIP_POOL=1 SKIP_RESOLVE=1 RIAK_TEST_HTTP_PORT=18098 ./tox_runner.sh .. + @RIAK_TEST_PROTOCOL='pbc' RUN_YZ=1 RUN_INDEXES=1 RUN_SECURITY=1 RUN_POOL=0 RUN_RESOLVE=0 ./tox_runner.sh .. + @RIAK_TEST_PROTOCOL='http' RUN_YZ=1 RUN_INDEXES=1 RUN_SECURITY=1 RUN_POOL=0 RUN_RESOLVE=0 RIAK_TEST_HTTP_PORT=18098 ./tox_runner.sh .. # These are required to actually build all the Python versions: # * pip install tox diff --git a/commands.py b/commands.py index 0f678c02..cd178e8b 100644 --- a/commands.py +++ b/commands.py @@ -74,6 +74,7 @@ def check_output(*popenargs, **kwargs): except ImportError: import json + class bucket_type_commands: def initialize_options(self): self.riak_admin = None @@ -143,6 +144,7 @@ def _btype_command(self, *args): cmd.extend(args) return cmd + class create_bucket_types(bucket_type_commands, Command): """ Creates bucket-types appropriate for testing. By default this will create: @@ -175,9 +177,14 @@ class create_bucket_types(bucket_type_commands, Command): class setup_timeseries(bucket_type_commands, Command): """ - Creates bucket-types appropriate for timeseries. By default this will create: - - * `GeoCheckin` with ``{"props": {"n_val": 3, "table_def": "CREATE TABLE GeoCheckin (geohash varchar not null, user varchar not null, time timestamp not null, weather varchar not null, temperature float, PRIMARY KEY((quantum(time, 15, m),user), time, user))"}}`` + Creates bucket-types appropriate for timeseries. + By default this will create: + + * `GeoCheckin` with ``{"props": {"n_val": 3, + "table_def": "CREATE TABLE GeoCheckin (geohash varchar not null, + user varchar not null, time timestamp not null, + weather varchar not null, temperature float, + PRIMARY KEY((quantum(time, 15, m),user), time, user))"}}`` """ description = "create bucket-types used in timeseries tests" @@ -187,7 +194,13 @@ class setup_timeseries(bucket_type_commands, Command): ] _props = { - 'GeoCheckin': {'n_val': 3, 'table_def': 'CREATE TABLE GeoCheckin (geohash varchar not null, user varchar not null, time timestamp not null, weather varchar not null, temperature float, PRIMARY KEY((quantum(time, 15, m),user), time, user))'}, + 'GeoCheckin': { + 'n_val': 3, + 'table_def': + 'CREATE TABLE GeoCheckin (geohash varchar not null, ' + + 'user varchar not null, time timestamp not null, ' + + 'weather varchar not null, temperature float, ' + + 'PRIMARY KEY((quantum(time, 15, m),user), time, user))'} } diff --git a/riak/table.py b/riak/table.py index 6635cd0a..ea30752b 100644 --- a/riak/table.py +++ b/riak/table.py @@ -1,5 +1,6 @@ from six import string_types, PY2 + class Table(object): """ The ``Table`` object allows you to access properties on a Riak @@ -36,7 +37,8 @@ def __repr__(self): def new(self, rows, columns=None): """ - A shortcut for manually instantiating a new :class:`~riak.ts_object.TsObject` + A shortcut for manually instantiating a new + :class:`~riak.ts_object.TsObject` :param rows: An list of lists with timeseries data :type rows: list diff --git a/riak/tests/__init__.py b/riak/tests/__init__.py index a7bb4742..f5aa6866 100644 --- a/riak/tests/__init__.py +++ b/riak/tests/__init__.py @@ -32,16 +32,17 @@ DUMMY_HTTP_PORT = int(os.environ.get('DUMMY_HTTP_PORT', '1023')) DUMMY_PB_PORT = int(os.environ.get('DUMMY_PB_PORT', '1022')) -SKIP_SEARCH = int(os.environ.get('SKIP_SEARCH', '1')) +RUN_SEARCH = int(os.environ.get('RUN_SEARCH', '0')) RUN_YZ = int(os.environ.get('RUN_YZ', '0')) -SKIP_INDEXES = int(os.environ.get('SKIP_INDEXES', '1')) +RUN_INDEXES = int(os.environ.get('RUN_INDEXES', '0')) -SKIP_TIMESERIES = int(os.environ.get('SKIP_TIMESERIES', '1')) +RUN_TIMESERIES = int(os.environ.get('RUN_TIMESERIES', '0')) -SKIP_POOL = int(os.environ.get('SKIP_POOL', '1')) -SKIP_RESOLVE = int(os.environ.get('SKIP_RESOLVE', '0')) -SKIP_BTYPES = int(os.environ.get('SKIP_BTYPES', '0')) +RUN_POOL = int(os.environ.get('RUN_POOL', '0')) +RUN_RESOLVE = int(os.environ.get('RUN_RESOLVE', '1')) +RUN_BTYPES = int(os.environ.get('RUN_BTYPES', '1')) +RUN_DATATYPES = int(os.environ.get('RUN_DATATYPES', '1')) RUN_SECURITY = int(os.environ.get('RUN_SECURITY', '0')) SECURITY_USER = os.environ.get('RIAK_TEST_SECURITY_USER', 'testuser') @@ -63,7 +64,9 @@ SECURITY_CERT_PASSWD = os.environ.get('RIAK_TEST_SECURITY_CERT_PASSWD', 'certpass') -SECURITY_CIPHERS = 'DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:AES128-SHA256:AES128-SHA:AES256-SHA256:AES256-SHA:RC4-SHA' +SECURITY_CIPHERS = 'DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:' + \ + 'DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:' + \ + 'AES128-SHA256:AES128-SHA:AES256-SHA256:AES256-SHA:RC4-SHA' SECURITY_CREDS = None if RUN_SECURITY: @@ -71,4 +74,3 @@ password=SECURITY_PASSWD, cacert_file=SECURITY_CACERT, ciphers=SECURITY_CIPHERS) -SKIP_DATATYPES = int(os.environ.get('SKIP_DATATYPES', '0')) diff --git a/riak/tests/base.py b/riak/tests/base.py index 8856f923..ec0f397c 100644 --- a/riak/tests/base.py +++ b/riak/tests/base.py @@ -7,6 +7,7 @@ from riak.client import RiakClient from riak.tests import HOST, PROTOCOL, PB_PORT, HTTP_PORT, SECURITY_CREDS + class IntegrationTestBase(object): host = None @@ -43,7 +44,15 @@ def create_client(cls, host=None, http_port=None, pb_port=None, credentials = credentials or SECURITY_CREDS if hasattr(cls, 'logging_enabled') and cls.logging_enabled: - cls.logger.debug("RiakClient(protocol='%s', host='%s', pb_port='%d', http_port='%d', credentials='%s', client_args='%s')", protocol, host, pb_port, http_port, credentials, client_args) + cls.logger.debug("RiakClient(protocol='%s', host='%s', " + + "pb_port='%d', http_port='%d', " + + "credentials='%s', client_args='%s')", + protocol, + host, + pb_port, + http_port, + credentials, + client_args) return RiakClient(protocol=protocol, host=host, diff --git a/riak/tests/test_2i.py b/riak/tests/test_2i.py index 419bce60..d7b254e3 100644 --- a/riak/tests/test_2i.py +++ b/riak/tests/test_2i.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import platform from riak import RiakError -from riak.tests import SKIP_INDEXES +from riak.tests import RUN_INDEXES from riak.tests.base import IntegrationTestBase if platform.python_version() < '2.7': @@ -21,7 +21,7 @@ def is_2i_supported(self): return False return True # it failed, but is supported! - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_secondary_index_store(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I not supported") @@ -102,7 +102,7 @@ def test_secondary_index_store(self): # Clean up... bucket.get('mykey1').delete() - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_set_indexes(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I not supported") @@ -120,7 +120,7 @@ def test_set_indexes(self): self.assertEqual(1, len(result)) self.assertEqual('foo', str(result[0])) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_remove_indexes(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I not supported") @@ -180,7 +180,7 @@ def test_remove_indexes(self): self.assertEqual(1, len([x for x in bar.indexes if x[0] == 'baz_bin'])) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_secondary_index_query(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I not supported") @@ -209,7 +209,7 @@ def test_secondary_index_query(self): self.assertEqual(3, len(results)) self.assertEqual(set([o2.key, o3.key, o4.key]), vals) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_secondary_index_invalid_name(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I not supported") @@ -219,7 +219,7 @@ def test_secondary_index_invalid_name(self): with self.assertRaises(RiakError): bucket.new('k', 'a').add_index('field1', 'value1') - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_set_index(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I not supported") @@ -237,7 +237,7 @@ def test_set_index(self): obj.set_index('bar2_int', 10) self.assertEqual(set((('bar_int', 3), ('bar2_int', 10))), obj.indexes) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_stream_index(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I not supported") @@ -250,7 +250,7 @@ def test_stream_index(self): self.assertEqual(sorted([o1.key, o2.key, o3.key]), sorted(keys)) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_index_return_terms(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I is not supported") @@ -274,7 +274,7 @@ def test_index_return_terms(self): self.assertEqual([(1002, o2.key), (1003, o3.key), (1004, o4.key)], sorted(spairs)) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_index_pagination(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I is not supported") @@ -309,7 +309,7 @@ def test_index_pagination(self): self.assertEqual(3, pagecount) self.assertEqual([o1.key, o2.key, o3.key, o4.key], presults) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_index_pagination_return_terms(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I is not supported") @@ -334,7 +334,7 @@ def test_index_pagination_return_terms(self): self.assertLessEqual(2, len(results)) self.assertEqual([('val3', o3.key), ('val4', o4.key)], page2) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_index_pagination_stream(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I is not supported") @@ -377,7 +377,7 @@ def test_index_pagination_stream(self): self.assertEqual(3, pagecount) self.assertEqual([o1.key, o2.key, o3.key, o4.key], presults) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_index_pagination_stream_return_terms(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I is not supported") @@ -409,7 +409,7 @@ def test_index_pagination_stream_return_terms(self): self.assertLessEqual(2, len(results)) self.assertEqual([('val3', o3.key), ('val4', o4.key)], results) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_index_eq_query_return_terms(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I is not supported") @@ -419,7 +419,7 @@ def test_index_eq_query_return_terms(self): results = bucket.get_index('field2_int', 1001, return_terms=True) self.assertEqual([(1001, o1.key)], results) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_index_eq_query_stream_return_terms(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I is not supported") @@ -432,7 +432,7 @@ def test_index_eq_query_stream_return_terms(self): self.assertEqual([(1001, o1.key)], results) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_index_timeout(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I is not supported") @@ -451,7 +451,7 @@ def test_index_timeout(self): self.assertEqual([o1.key], bucket.get_index('field1_bin', 'val1', timeout='infinity')) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_index_regex(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I is not supported") @@ -466,7 +466,7 @@ def test_index_regex(self): self.assertEqual([('val2', o2.key)], results) - @unittest.skipIf(SKIP_INDEXES, 'SKIP_INDEXES is defined') + @unittest.skipUnless(RUN_INDEXES, 'RUN_INDEXES is 0') def test_index_falsey_endkey_gh378(self): if not self.is_2i_supported(): raise unittest.SkipTest("2I is not supported") diff --git a/riak/tests/test_btypes.py b/riak/tests/test_btypes.py index c1b57f6c..3c8b6c1e 100644 --- a/riak/tests/test_btypes.py +++ b/riak/tests/test_btypes.py @@ -1,7 +1,7 @@ import platform from riak import RiakError, RiakObject from riak.bucket import RiakBucket, BucketType -from riak.tests import SKIP_BTYPES +from riak.tests import RUN_BTYPES from riak.tests.base import IntegrationTestBase from riak.tests.comparison import Comparison @@ -11,6 +11,7 @@ import unittest +@unittest.skipUnless(RUN_BTYPES, "RUN_BTYPES is 0") class BucketTypeTests(IntegrationTestBase, unittest.TestCase, Comparison): def test_btype_init(self): btype = self.client.bucket_type('foo') @@ -41,7 +42,6 @@ def test_btype_repr(self): self.assertEqual("", repr(defbtype)) self.assertEqual("", repr(othertype)) - @unittest.skipIf(SKIP_BTYPES == '1', "SKIP_BTYPES is set") def test_btype_get_props(self): defbtype = self.client.bucket_type("default") btype = self.client.bucket_type("pytest") @@ -53,7 +53,6 @@ def test_btype_get_props(self): self.assertIn('n_val', props) self.assertEqual(3, props['n_val']) - @unittest.skipIf(SKIP_BTYPES == '1', "SKIP_BTYPES is set") def test_btype_set_props(self): defbtype = self.client.bucket_type("default") btype = self.client.bucket_type("pytest") @@ -72,13 +71,11 @@ def test_btype_set_props(self): finally: btype.set_properties(oldprops) - @unittest.skipIf(SKIP_BTYPES == '1', "SKIP_BTYPES is set") def test_btype_set_props_immutable(self): btype = self.client.bucket_type("pytest-maps") with self.assertRaises(RiakError): btype.set_property('datatype', 'counter') - @unittest.skipIf(SKIP_BTYPES == '1', "SKIP_BTYPES is set") def test_btype_list_buckets(self): btype = self.client.bucket_type("pytest") bucket = btype.bucket(self.bucket_name) @@ -93,7 +90,6 @@ def test_btype_list_buckets(self): self.assertIn(bucket, buckets) - @unittest.skipIf(SKIP_BTYPES == '1', "SKIP_BTYPES is set") def test_btype_list_keys(self): btype = self.client.bucket_type("pytest") bucket = btype.bucket(self.bucket_name) @@ -109,7 +105,6 @@ def test_btype_list_keys(self): self.assertIn(self.key_name, keys) - @unittest.skipIf(SKIP_BTYPES == '1', "SKIP_BTYPES is set") def test_default_btype_list_buckets(self): default_btype = self.client.bucket_type("default") bucket = default_btype.bucket(self.bucket_name) @@ -126,7 +121,6 @@ def test_default_btype_list_buckets(self): self.assertItemsEqual(buckets, self.client.get_buckets()) - @unittest.skipIf(SKIP_BTYPES == '1', "SKIP_BTYPES is set") def test_default_btype_list_keys(self): btype = self.client.bucket_type("default") bucket = btype.bucket(self.bucket_name) @@ -145,7 +139,6 @@ def test_default_btype_list_keys(self): oldapikeys = self.client.get_keys(self.client.bucket(self.bucket_name)) self.assertItemsEqual(keys, oldapikeys) - @unittest.skipIf(SKIP_BTYPES == '1', "SKIP_BTYPES is set") def test_multiget_bucket_types(self): btype = self.client.bucket_type('pytest') bucket = btype.bucket(self.bucket_name) @@ -161,7 +154,6 @@ def test_multiget_bucket_types(self): self.assertEqual(bucket, mobj.bucket) self.assertEqual(btype, mobj.bucket.bucket_type) - @unittest.skipIf(SKIP_BTYPES == '1', "SKIP_BTYPES is set") def test_write_once_bucket_type(self): btype = self.client.bucket_type('pytest-write-once') btype.set_property('write_once', True) diff --git a/riak/tests/test_client.py b/riak/tests/test_client.py index be105502..46700a61 100644 --- a/riak/tests/test_client.py +++ b/riak/tests/test_client.py @@ -2,7 +2,7 @@ from six import PY2 from threading import Thread from riak.riak_object import RiakObject -from riak.tests import DUMMY_HTTP_PORT, DUMMY_PB_PORT, SKIP_POOL +from riak.tests import DUMMY_HTTP_PORT, DUMMY_PB_PORT, RUN_POOL from riak.tests.base import IntegrationTestBase if PY2: @@ -15,6 +15,7 @@ else: import unittest + class ClientTests(IntegrationTestBase, unittest.TestCase): def test_uses_client_id_if_given(self): if self.protocol == 'pbc': @@ -192,7 +193,7 @@ def test_multiget_pool_size(self): self.assertEqual(obj.key, obj.data) client.close() - @unittest.skipIf(SKIP_POOL, 'SKIP_POOL is set') + @unittest.skipUnless(RUN_POOL, 'RUN_POOL is 0') def test_pool_close(self): """ Iterate over the connection pool and close all connections. diff --git a/riak/tests/test_datatypes.py b/riak/tests/test_datatypes.py index b8ca881f..747de515 100644 --- a/riak/tests/test_datatypes.py +++ b/riak/tests/test_datatypes.py @@ -2,7 +2,7 @@ import platform from riak import RiakBucket, BucketType, RiakObject import riak.datatypes as datatypes -from riak.tests import SKIP_DATATYPES +from riak.tests import RUN_DATATYPES from riak.tests.base import IntegrationTestBase from riak.tests.comparison import Comparison @@ -147,8 +147,10 @@ def test_removes_require_context(self): self.assertTrue(dtype.modified) -class DatatypeIntegrationTests(IntegrationTestBase, unittest.TestCase, Comparison): - @unittest.skipIf(SKIP_DATATYPES, 'SKIP_DATATYPES is set') +@unittest.skipUnless(RUN_DATATYPES, 'RUN_DATATYPES is 0') +class DatatypeIntegrationTests(IntegrationTestBase, + unittest.TestCase, + Comparison): def test_dt_counter(self): btype = self.client.bucket_type('pytest-counters') bucket = btype.bucket(self.bucket_name) @@ -165,7 +167,6 @@ def test_dt_counter(self): mycount.reload() self.assertEqual(2, mycount.value) - @unittest.skipIf(SKIP_DATATYPES, 'SKIP_DATATYPES is set') def test_dt_set(self): btype = self.client.bucket_type('pytest-sets') bucket = btype.bucket(self.bucket_name) @@ -188,7 +189,6 @@ def test_dt_set(self): self.assertIn('Brett', myset) self.assertNotIn('Sean', myset) - @unittest.skipIf(SKIP_DATATYPES, 'SKIP_DATATYPES is set') def test_dt_map(self): btype = self.client.bucket_type('pytest-maps') bucket = btype.bucket(self.bucket_name) @@ -224,7 +224,6 @@ def test_dt_map(self): self.assertIn('f', mymap.sets) self.assertItemsEqual(['thing1', 'thing2'], mymap.sets['f'].value) - @unittest.skipIf(SKIP_DATATYPES, 'SKIP_DATATYPES is set') def test_dt_set_remove_without_context(self): btype = self.client.bucket_type('pytest-sets') bucket = btype.bucket(self.bucket_name) @@ -236,7 +235,6 @@ def test_dt_set_remove_without_context(self): with self.assertRaises(datatypes.ContextRequired): set.discard("Y") - @unittest.skipIf(SKIP_DATATYPES, 'SKIP_DATATYPES is set') def test_dt_set_remove_fetching_context(self): btype = self.client.bucket_type('pytest-sets') bucket = btype.bucket(self.bucket_name) @@ -253,7 +251,6 @@ def test_dt_set_remove_fetching_context(self): set2 = bucket.get(self.key_name) self.assertItemsEqual(['X', 'Y'], set2.value) - @unittest.skipIf(SKIP_DATATYPES, 'SKIP_DATATYPES is set') def test_dt_set_add_twice(self): btype = self.client.bucket_type('pytest-sets') bucket = btype.bucket(self.bucket_name) @@ -270,7 +267,6 @@ def test_dt_set_add_twice(self): set2 = bucket.get(self.key_name) self.assertItemsEqual(['X', 'Y'], set2.value) - @unittest.skipIf(SKIP_DATATYPES, 'SKIP_DATATYPES is set') def test_dt_set_add_wins_in_same_op(self): btype = self.client.bucket_type('pytest-sets') bucket = btype.bucket(self.bucket_name) @@ -288,7 +284,6 @@ def test_dt_set_add_wins_in_same_op(self): set2 = bucket.get(self.key_name) self.assertItemsEqual(['X', 'Y'], set2.value) - @unittest.skipIf(SKIP_DATATYPES, 'SKIP_DATATYPES is set') def test_dt_set_add_wins_in_same_op_reversed(self): btype = self.client.bucket_type('pytest-sets') bucket = btype.bucket(self.bucket_name) @@ -306,7 +301,6 @@ def test_dt_set_add_wins_in_same_op_reversed(self): set2 = bucket.get(self.key_name) self.assertItemsEqual(['X', 'Y'], set2.value) - @unittest.skipIf(SKIP_DATATYPES, 'SKIP_DATATYPES is set') def test_dt_set_remove_old_context(self): btype = self.client.bucket_type('pytest-sets') bucket = btype.bucket(self.bucket_name) @@ -328,7 +322,6 @@ def test_dt_set_remove_old_context(self): set2 = bucket.get(self.key_name) self.assertItemsEqual(['X', 'Y', 'Z'], set2.value) - @unittest.skipIf(SKIP_DATATYPES, 'SKIP_DATATYPES is set') def test_dt_set_remove_updated_context(self): btype = self.client.bucket_type('pytest-sets') bucket = btype.bucket(self.bucket_name) @@ -349,7 +342,6 @@ def test_dt_set_remove_updated_context(self): set2 = bucket.get(self.key_name) self.assertItemsEqual(['X', 'Y'], set2.value) - @unittest.skipIf(SKIP_DATATYPES, 'SKIP_DATATYPES is set') def test_dt_map_remove_set_update_same_op(self): btype = self.client.bucket_type('pytest-maps') bucket = btype.bucket(self.bucket_name) @@ -367,7 +359,6 @@ def test_dt_map_remove_set_update_same_op(self): map2 = bucket.get(self.key_name) self.assertItemsEqual(["Z"], map2.sets['set']) - @unittest.skipIf(SKIP_DATATYPES, 'SKIP_DATATYPES is set') def test_dt_map_remove_counter_increment_same_op(self): btype = self.client.bucket_type('pytest-maps') bucket = btype.bucket(self.bucket_name) @@ -385,7 +376,6 @@ def test_dt_map_remove_counter_increment_same_op(self): map2 = bucket.get(self.key_name) self.assertEqual(2, map2.counters['counter'].value) - @unittest.skipIf(SKIP_DATATYPES, 'SKIP_DATATYPES is set') def test_dt_map_remove_map_update_same_op(self): btype = self.client.bucket_type('pytest-maps') bucket = btype.bucket(self.bucket_name) @@ -403,7 +393,6 @@ def test_dt_map_remove_map_update_same_op(self): map2 = bucket.get(self.key_name) self.assertItemsEqual(["Z"], map2.maps['map'].sets['set']) - @unittest.skipIf(SKIP_DATATYPES, 'SKIP_DATATYPES is set') def test_dt_set_return_body_true_default(self): btype = self.client.bucket_type('pytest-sets') bucket = btype.bucket(self.bucket_name) @@ -421,7 +410,6 @@ def test_dt_set_return_body_true_default(self): myset.store() self.assertItemsEqual(myset.value, ['Y']) - @unittest.skipIf(SKIP_DATATYPES, 'SKIP_DATATYPES is set') def test_dt_map_return_body_true_default(self): btype = self.client.bucket_type('pytest-maps') bucket = btype.bucket(self.bucket_name) @@ -446,7 +434,6 @@ def test_dt_map_return_body_true_default(self): self.assertEqual(mymap.value, {}) - @unittest.skipIf(SKIP_DATATYPES, 'SKIP_DATATYPES is set') def test_delete_datatype(self): ctype = self.client.bucket_type('pytest-counters') cbucket = ctype.bucket(self.bucket_name) diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index 592031c5..cc7b20c9 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -7,7 +7,7 @@ from time import sleep from riak import ConflictError, RiakBucket, RiakError from riak.resolver import default_resolver, last_written_resolver -from riak.tests import SKIP_RESOLVE +from riak.tests import RUN_RESOLVE from riak.tests.base import IntegrationTestBase from riak.tests.comparison import Comparison @@ -34,11 +34,13 @@ testrun_sibs_bucket = 'sibsbucket' testrun_props_bucket = 'propsbucket' + def setUpModule(): c = IntegrationTestBase.create_client() c.bucket(testrun_sibs_bucket).allow_mult = True c.close() + def tearDownModule(): c = IntegrationTestBase.create_client() c.bucket(testrun_sibs_bucket).clear_properties() @@ -430,8 +432,7 @@ def test_siblings(self): self.assertEqual(len(obj.siblings), 1) self.assertEqual(obj.data, resolved_sibling.data) - @unittest.skipIf(SKIP_RESOLVE == '1', - "skip requested for resolvers test") + @unittest.skipUnless(RUN_RESOLVE, "RUN_RESOLVE is 0") def test_resolution(self): bucket = self.client.bucket(testrun_sibs_bucket) obj = bucket.get(self.key_name) @@ -487,8 +488,7 @@ def max_value_resolver(obj): self.assertEqual(bucket.resolver, default_resolver) # reset self.assertEqual(self.client.resolver, default_resolver) # reset - @unittest.skipIf(SKIP_RESOLVE == '1', - "skip requested for resolvers test") + @unittest.skipUnless(RUN_RESOLVE, "RUN_RESOLVE is 0") def test_resolution_default(self): # If no resolver is setup, be sure to resolve to default_resolver bucket = self.client.bucket(testrun_sibs_bucket) diff --git a/riak/tests/test_mapreduce.py b/riak/tests/test_mapreduce.py index e0897f6a..b6cd068f 100644 --- a/riak/tests/test_mapreduce.py +++ b/riak/tests/test_mapreduce.py @@ -18,11 +18,15 @@ import unittest -testrun_yz_mr = {'btype': 'pytest-mr', 'bucket': 'mrbucket', 'index': 'mrbucket'} +testrun_yz_mr = {'btype': 'pytest-mr', + 'bucket': 'mrbucket', + 'index': 'mrbucket'} + def setUpModule(): yzSetUp(testrun_yz_mr) + def tearDownModule(): yzTearDown(testrun_yz_mr) @@ -516,7 +520,7 @@ def test_mr_list_add_mix(self): u'"fooval2"', u'"fooval3"']) - @unittest.skipUnless(RUN_YZ, 'RUN_YZ is undefined') + @unittest.skipUnless(RUN_YZ, 'RUN_YZ is 0') def test_mr_search(self): """ Try a successful map/reduce from search results. diff --git a/riak/tests/test_pool.py b/riak/tests/test_pool.py index edfba3f5..f1088244 100644 --- a/riak/tests/test_pool.py +++ b/riak/tests/test_pool.py @@ -5,7 +5,7 @@ from riak.transports.pool import Pool, BadResource from random import SystemRandom from time import sleep -from riak.tests import SKIP_POOL +from riak.tests import RUN_POOL from riak.tests.comparison import Comparison if platform.python_version() < '2.7': @@ -37,7 +37,7 @@ def create_resource(self): return [] -@unittest.skipIf(SKIP_POOL, 'Skipping connection pool tests') +@unittest.skipUnless(RUN_POOL, 'RUN_POOL is 0') class PoolTest(unittest.TestCase, Comparison): def test_yields_new_object_when_empty(self): diff --git a/riak/tests/test_search.py b/riak/tests/test_search.py index 73c6cd47..7cc369b6 100644 --- a/riak/tests/test_search.py +++ b/riak/tests/test_search.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from __future__ import print_function import platform -from riak.tests import SKIP_SEARCH +from riak.tests import RUN_SEARCH, RUN_YZ from riak.tests.base import IntegrationTestBase if platform.python_version() < '2.7': @@ -11,27 +11,29 @@ testrun_search_bucket = 'searchbucket' + def setUpModule(): - if not SKIP_SEARCH and not RUN_YZ: + if RUN_SEARCH and not RUN_YZ: c = IntegrationTestBase.create_client() b = c.bucket(testrun_search_bucket) b.enable_search() c.close() + def tearDownModule(): - if not SKIP_SEARCH and not RUN_YZ: + if RUN_SEARCH and not RUN_YZ: c = IntegrationTestBase.create_client() b = c.bucket(testrun_search_bucket) b.clear_properties() c.close() + +@unittest.skipUnless(RUN_SEARCH, 'RUN_SEARCH is 0') class EnableSearchTests(IntegrationTestBase, unittest.TestCase): - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_bucket_search_enabled(self): bucket = self.client.bucket(self.bucket_name) self.assertFalse(bucket.search_enabled()) - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_enable_search_commit_hook(self): bucket = self.client.bucket(testrun_search_bucket) bucket.clear_properties() @@ -46,7 +48,6 @@ def test_enable_search_commit_hook(self): self.assertTrue(c.bucket(testrun_search_bucket).search_enabled()) c.close() - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_disable_search_commit_hook(self): bucket = self.client.bucket(testrun_search_bucket) bucket.clear_properties() @@ -65,8 +66,8 @@ def test_disable_search_commit_hook(self): bucket.enable_search() +@unittest.skipUnless(RUN_SEARCH, 'RUN_SEARCH is 0') class SolrSearchTests(IntegrationTestBase, unittest.TestCase): - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_add_document_to_index(self): self.client.fulltext_add(testrun_search_bucket, [{"id": "doc", "username": "tony"}]) @@ -74,7 +75,6 @@ def test_add_document_to_index(self): "username:tony") self.assertEqual("tony", results['docs'][0]['username']) - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_add_multiple_documents_to_index(self): self.client.fulltext_add( testrun_search_bucket, @@ -84,7 +84,6 @@ def test_add_multiple_documents_to_index(self): testrun_search_bucket, "username:russell OR username:dizzy") self.assertEqual(2, len(results['docs'])) - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_id(self): self.client.fulltext_add( testrun_search_bucket, @@ -95,7 +94,6 @@ def test_delete_documents_from_search_by_id(self): testrun_search_bucket, "username:russell OR username:dizzy") self.assertEqual(1, len(results['docs'])) - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_query(self): self.client.fulltext_add( testrun_search_bucket, @@ -108,7 +106,6 @@ def test_delete_documents_from_search_by_query(self): testrun_search_bucket, "username:russell OR username:dizzy") self.assertEqual(0, len(results['docs'])) - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_delete_documents_from_search_by_query_and_id(self): self.client.fulltext_add( testrun_search_bucket, @@ -124,22 +121,20 @@ def test_delete_documents_from_search_by_query_and_id(self): self.assertEqual(0, len(results['docs'])) +@unittest.skipUnless(RUN_SEARCH, 'RUN_SEARCH is 0') class SearchTests(IntegrationTestBase, unittest.TestCase): - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_solr_search_from_bucket(self): bucket = self.client.bucket(testrun_search_bucket) bucket.new("user", {"username": "roidrage"}).store() results = bucket.search("username:roidrage") self.assertEqual(1, len(results['docs'])) - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_solr_search_with_params_from_bucket(self): bucket = self.client.bucket(testrun_search_bucket) bucket.new("user", {"username": "roidrage"}).store() results = bucket.search("username:roidrage", wt="xml") self.assertEqual(1, len(results['docs'])) - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_solr_search_with_params(self): bucket = self.client.bucket(testrun_search_bucket) bucket.new("user", {"username": "roidrage"}).store() @@ -148,7 +143,6 @@ def test_solr_search_with_params(self): "username:roidrage", wt="xml") self.assertEqual(1, len(results['docs'])) - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_solr_search(self): bucket = self.client.bucket(testrun_search_bucket) bucket.new("user", {"username": "roidrage"}).store() @@ -156,7 +150,6 @@ def test_solr_search(self): "username:roidrage") self.assertEqual(1, len(results["docs"])) - @unittest.skipIf(SKIP_SEARCH, 'SKIP_SEARCH is defined') def test_search_integration(self): # Create some objects to search across... bucket = self.client.bucket(testrun_search_bucket) diff --git a/riak/tests/test_security.py b/riak/tests/test_security.py index 0c8a5a49..135bb112 100644 --- a/riak/tests/test_security.py +++ b/riak/tests/test_security.py @@ -16,14 +16,14 @@ class SecurityTests(IntegrationTestBase, unittest.TestCase): - @unittest.skipIf(RUN_SECURITY, 'RUN_SECURITY is set') + @unittest.skipIf(RUN_SECURITY, 'RUN_SECURITY is 1') def test_security_disabled(self): - topts = { 'timeout' : 1 } - creds = SecurityCreds(username='foo', - password='bar', - cacert_file=SECURITY_CACERT, - ciphers=SECURITY_CIPHERS) - client = self.create_client(credentials=creds, transport_options=topts) + """ + Test valid security settings without security enabled + """ + topts = {'timeout': 1} + client = self.create_client(credentials=SECURITY_CREDS, + transport_options=topts) myBucket = client.bucket('test') val1 = "foobar" key1 = myBucket.new('x', data=val1) @@ -31,7 +31,7 @@ def test_security_disabled(self): key1.store() client.close() - @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') + @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is 0') def test_security_basic_connection(self): myBucket = self.client.bucket('test') val1 = "foobar" @@ -39,7 +39,7 @@ def test_security_basic_connection(self): key1.store() myBucket.get('x') - @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') + @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is 0') def test_security_bad_user(self): creds = SecurityCreds(username='foo', password=SECURITY_PASSWD, @@ -50,7 +50,7 @@ def test_security_bad_user(self): client.get_buckets() client.close() - @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') + @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is 0') def test_security_bad_password(self): creds = SecurityCreds(username=SECURITY_USER, password='foo', @@ -61,7 +61,7 @@ def test_security_bad_password(self): client.get_buckets() client.close() - @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') + @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is 0') def test_security_invalid_cert(self): creds = SecurityCreds(username=SECURITY_USER, password=SECURITY_PASSWD, @@ -72,7 +72,7 @@ def test_security_invalid_cert(self): client.get_buckets() client.close() - @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') + @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is 0') def test_security_password_without_cacert(self): creds = SecurityCreds(username=SECURITY_USER, password=SECURITY_PASSWD, @@ -85,7 +85,7 @@ def test_security_password_without_cacert(self): key1.store() client.close() - @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') + @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is 0') def test_security_cert_authentication(self): creds = SecurityCreds(username=SECURITY_CERT_USER, password=SECURITY_CERT_PASSWD, @@ -108,9 +108,10 @@ def test_security_cert_authentication(self): myBucket.get('x') client.close() - @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') + @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is 0') def test_security_revoked_cert(self): - creds = SecurityCreds(username=SECURITY_USER, password=SECURITY_PASSWD, + creds = SecurityCreds(username=SECURITY_USER, + password=SECURITY_PASSWD, ciphers=SECURITY_CIPHERS, cacert_file=SECURITY_CACERT, crl_file=SECURITY_REVOKED) @@ -123,7 +124,7 @@ def test_security_revoked_cert(self): client.get_buckets() client.close() - @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') + @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is 0') def test_security_bad_ca_cert(self): creds = SecurityCreds(username=SECURITY_USER, password=SECURITY_PASSWD, ciphers=SECURITY_CIPHERS, @@ -133,7 +134,7 @@ def test_security_bad_ca_cert(self): client.get_buckets() client.close() - @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') + @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is 0') def test_security_ciphers(self): creds = SecurityCreds(username=SECURITY_USER, password=SECURITY_PASSWD, ciphers=SECURITY_CIPHERS, @@ -146,7 +147,7 @@ def test_security_ciphers(self): myBucket.get('x') client.close() - @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is not set') + @unittest.skipUnless(RUN_SECURITY, 'RUN_SECURITY is 0') def test_security_bad_ciphers(self): creds = SecurityCreds(username=SECURITY_USER, password=SECURITY_PASSWD, cacert_file=SECURITY_CACERT, diff --git a/riak/tests/test_timeseries.py b/riak/tests/test_timeseries.py index dbc1674d..d6f63c07 100644 --- a/riak/tests/test_timeseries.py +++ b/riak/tests/test_timeseries.py @@ -3,14 +3,12 @@ import os import platform import riak_pb -import sys -import time from riak.table import Table from riak.ts_object import TsObject from riak.transports.pbc.codec import RiakPbcCodec from riak.util import str_to_bytes, bytes_to_str -from riak.tests import SKIP_TIMESERIES +from riak.tests import RUN_TIMESERIES from riak.tests.base import IntegrationTestBase if platform.python_version() < '2.7': @@ -27,18 +25,20 @@ ts0 = datetime.datetime(2015, 1, 1, 12, 0, 0) ts1 = ts0 + fiveMins + +@unittest.skipUnless(RUN_TIMESERIES, 'RUN_TIMESERIES is 0') class TimeseriesUnitTests(unittest.TestCase): def setUp(self): self.c = RiakPbcCodec() self.ts0ms = self.c._unix_time_millis(ts0) self.ts1ms = self.c._unix_time_millis(ts1) self.rows = [ - [ bd0, 0, 1.2, ts0, True ], - [ bd1, 3, 4.5, ts1, False ] + [bd0, 0, 1.2, ts0, True], + [bd1, 3, 4.5, ts1, False] ] self.test_key = { - 'user' : 'user2', - 'time' : ts0 + 'user': 'user2', + 'time': ts0 } self.table = Table(None, 'test-table') @@ -157,7 +157,8 @@ def test_decode_data_from_query(self): self.assertEqual(r1[3], ts1) self.assertEqual(r1[4], self.rows[1][4]) -@unittest.skipIf(SKIP_TIMESERIES == 1, "skip requested for timeseries tests") + +@unittest.skipUnless(RUN_TIMESERIES, 'RUN_TIMESERIES is 0') class TimeseriesTests(IntegrationTestBase, unittest.TestCase): @classmethod def setUpClass(cls): @@ -179,14 +180,14 @@ def setUpClass(cls): # PRIMARY KEY((quantum(time, 15, m), user), time, user) # ) rows = [ - [ 'hash1', 'user2', twentyMinsAgo, 'hurricane', 82.3 ], - [ 'hash1', 'user2', fifteenMinsAgo, 'rain', 79.0 ], - [ 'hash1', 'user2', fiveMinsAgo, 'wind', None ], - [ 'hash1', 'user2', cls.now, 'snow', 20.1 ] + ['hash1', 'user2', twentyMinsAgo, 'hurricane', 82.3], + ['hash1', 'user2', fifteenMinsAgo, 'rain', 79.0], + ['hash1', 'user2', fiveMinsAgo, 'wind', None], + ['hash1', 'user2', cls.now, 'snow', 20.1] ] ts_obj = table.new(rows) result = ts_obj.store() - if result != True: + if not result: raise AssertionError("expected success") client.close() @@ -207,33 +208,36 @@ def validate_data(self, ts_obj): self.assertIsNone(row[4]) def test_query_that_returns_no_data(self): - query = "select * from {} where time > 0 and time < 10 and user = 'user1'".format(table_name) + query = "select * from {} where time > 0 and " + \ + "time < 10 and user = 'user1'".format(table_name) ts_obj = self.client.ts_query('GeoCheckin', query) self.assertEqual(len(ts_obj.columns), 0) self.assertEqual(len(ts_obj.rows), 0) def test_query_that_matches_some_data(self): - query = "select * from {} where time > {} and time < {} and user = 'user2'".format(table_name, self.tenMinsAgoMsec, self.nowMsec) + query = "select * from {} where time > {} and " + \ + " time < {} and user = 'user2'" \ + .format(table_name, self.tenMinsAgoMsec, self.nowMsec) ts_obj = self.client.ts_query('GeoCheckin', query) self.validate_data(ts_obj) def test_get_single_value_using_dict(self): key = { - 'user' : 'user2', - 'time' : self.fiveMinsAgo + 'user': 'user2', + 'time': self.fiveMinsAgo } ts_obj = self.client.ts_get('GeoCheckin', key) self.validate_data(ts_obj) def test_get_single_value_using_array(self): - key = [ self.fiveMinsAgo, 'user2' ] + key = [self.fiveMinsAgo, 'user2'] ts_obj = self.client.ts_get('GeoCheckin', key) self.validate_data(ts_obj) def test_delete_single_value_using_dict(self): key = { - 'user' : 'user2', - 'time' : self.twentyMinsAgo + 'user': 'user2', + 'time': self.twentyMinsAgo } rslt = self.client.ts_delete('GeoCheckin', key) self.assertTrue(rslt) diff --git a/riak/tests/test_yokozuna.py b/riak/tests/test_yokozuna.py index 55399aac..52f9af88 100644 --- a/riak/tests/test_yokozuna.py +++ b/riak/tests/test_yokozuna.py @@ -10,6 +10,7 @@ else: import unittest + def wait_for_yz_index(bucket, key, index=None): """ Wait until Solr index has been updated and a value returns from a query. @@ -25,16 +26,21 @@ def wait_for_yz_index(bucket, key, index=None): # YZ index on bucket of the same name testrun_yz = {'btype': None, 'bucket': 'yzbucket', 'index': 'yzbucket'} # YZ index on bucket of a different name -testrun_yz_index = {'btype': None, 'bucket': 'yzindexbucket', 'index': 'yzindex'} +testrun_yz_index = {'btype': None, + 'bucket': 'yzindexbucket', + 'index': 'yzindex'} + def setUpModule(): yzSetUp(testrun_yz, testrun_yz_index) + def tearDownModule(): yzTearDown(testrun_yz, testrun_yz_index) + +@unittest.skipUnless(RUN_YZ, 'RUN_YZ is 0') class YZSearchTests(IntegrationTestBase, unittest.TestCase, Comparison): - @unittest.skipUnless(RUN_YZ, 'RUN_YZ is undefined') def test_yz_search_from_bucket(self): bucket = self.client.bucket(testrun_yz['bucket']) bucket.new("user", {"user_s": "Z"}).store() @@ -51,16 +57,15 @@ def test_yz_search_from_bucket(self): self.assertIn('user_s', result) self.assertEqual(u'Z', result['user_s']) - @unittest.skipUnless(RUN_YZ, 'RUN_YZ is undefined') def test_yz_search_index_using_bucket(self): bucket = self.client.bucket(testrun_yz_index['bucket']) bucket.new("feliz", {"name_s": "Felix", "species_s": "Felis catus"}).store() wait_for_yz_index(bucket, "feliz", index=testrun_yz_index['index']) - results = bucket.search('name_s:Felix', index=testrun_yz_index['index']) + results = bucket.search('name_s:Felix', + index=testrun_yz_index['index']) self.assertEqual(1, len(results['docs'])) - @unittest.skipUnless(RUN_YZ, 'RUN_YZ is undefined') def test_yz_search_index_using_wrong_bucket(self): bucket = self.client.bucket(testrun_yz_index['bucket']) bucket.new("feliz", @@ -69,7 +74,6 @@ def test_yz_search_index_using_wrong_bucket(self): with self.assertRaises(Exception): bucket.search('name_s:Felix') - @unittest.skipUnless(RUN_YZ, 'RUN_YZ is undefined') def test_yz_get_search_index(self): index = self.client.get_search_index(testrun_yz['bucket']) self.assertEqual(testrun_yz['bucket'], index['name']) @@ -78,7 +82,6 @@ def test_yz_get_search_index(self): with self.assertRaises(Exception): self.client.get_search_index('NOT' + testrun_yz['bucket']) - @unittest.skipUnless(RUN_YZ, 'RUN_YZ is undefined') def test_yz_delete_search_index(self): # expected to fail, since there's an attached bucket with self.assertRaises(Exception): @@ -96,13 +99,11 @@ def test_yz_delete_search_index(self): while testrun_yz['bucket'] not in indexes: indexes = [i['name'] for i in self.client.list_search_indexes()] - @unittest.skipUnless(RUN_YZ, 'RUN_YZ is undefined') def test_yz_list_search_indexes(self): indexes = self.client.list_search_indexes() self.assertIn(testrun_yz['bucket'], [item['name'] for item in indexes]) self.assertLessEqual(1, len(indexes)) - @unittest.skipUnless(RUN_YZ, 'RUN_YZ is undefined') def test_yz_create_schema(self): content = """ @@ -138,7 +139,6 @@ def test_yz_create_schema(self): self.assertEqual(schema_name, schema['name']) self.assertEqual(content, schema['content']) - @unittest.skipUnless(RUN_YZ, 'RUN_YZ is undefined') def test_yz_create_bad_schema(self): bad_content = """ timestamp: '%s', timestamp_value '%d'", - cell, ts_cell.timestamp_value) + cell, ts_cell.timestamp_value) elif isinstance(cell, bool): logging.debug("cell -> boolean: '%s'", cell) ts_cell.boolean_value = cell elif isinstance(cell, str): logging.debug("cell -> str: '%s'", cell) ts_cell.binary_value = str_to_bytes(cell) - elif isinstance(cell, int) or isinstance(cell, long): + elif isinstance(cell, int) or isinstance(cell, long): # noqa logging.debug("cell -> int/long: '%s'", cell) ts_cell.integer_value = cell elif isinstance(cell, float): @@ -645,7 +648,8 @@ def _encode_to_ts_cell(self, cell, ts_cell): ts_cell.double_value = cell else: t = type(cell) - raise RiakError("can't serialize type '{}', value '{}'".format(t, cell)) + raise RiakError("can't serialize type '{}', value '{}'" + .format(t, cell)) def _encode_timeseries_keyreq(self, table, key, req): key_vals = None @@ -678,11 +682,11 @@ def _encode_timeseries_put(self, tsobj, ts_put_req): if tsobj.rows and isinstance(tsobj.rows, list): for row in tsobj.rows: - tsr = ts_put_req.rows.add() # NB: type riak_pb.TsRow + tsr = ts_put_req.rows.add() # NB: type riak_pb.TsRow if not isinstance(row, list): raise ValueError("TsObject row must be a list of values") for cell in row: - tsc = tsr.cells.add() # NB: type riak_pb.TsCell + tsc = tsr.cells.add() # NB: type riak_pb.TsCell self._encode_to_ts_cell(cell, tsc) else: raise RiakError("TsObject requires a list of rows") @@ -706,7 +710,8 @@ def _decode_timeseries(self, ts_rsp, tsobj): tsobj.columns.append(col) for ts_row in ts_rsp.rows: - tsobj.rows.append(self._decode_timeseries_row(ts_row, ts_rsp.columns)) + tsobj.rows.append(self._decode_timeseries_row(ts_row, + ts_rsp.columns)) def _decode_timeseries_row(self, ts_row, ts_columns): """ @@ -722,13 +727,18 @@ def _decode_timeseries_row(self, ts_row, ts_columns): for i, ts_cell in enumerate(ts_row.cells): ts_col = ts_columns[i] logging.debug("ts_cell: '%s', ts_col: '%d'", ts_cell, ts_col.type) - if ts_col.type == riak_pb.TsColumnType.Value('BINARY') and ts_cell.HasField('binary_value'): - logging.debug("ts_cell.binary_value: '%s'", ts_cell.binary_value) + if ts_col.type == riak_pb.TsColumnType.Value('BINARY')\ + and ts_cell.HasField('binary_value'): + logging.debug("ts_cell.binary_value: '%s'", + ts_cell.binary_value) row.append(ts_cell.binary_value) - elif ts_col.type == riak_pb.TsColumnType.Value('INTEGER') and ts_cell.HasField('integer_value'): - logging.debug("ts_cell.integer_value: '%s'", ts_cell.integer_value) + elif ts_col.type == riak_pb.TsColumnType.Value('INTEGER')\ + and ts_cell.HasField('integer_value'): + logging.debug("ts_cell.integer_value: '%s'", + ts_cell.integer_value) row.append(ts_cell.integer_value) - elif ts_col.type == riak_pb.TsColumnType.Value('FLOAT') and ts_cell.HasField('double_value'): + elif ts_col.type == riak_pb.TsColumnType.Value('FLOAT')\ + and ts_cell.HasField('double_value'): value = None if ts_cell.HasField('double_value'): value = ts_cell.double_value @@ -739,13 +749,17 @@ def _decode_timeseries_row(self, ts_row, ts_columns): elif ts_col.type == riak_pb.TsColumnType.Value('TIMESTAMP'): dt = None if ts_cell.HasField('timestamp_value'): - dt = self._datetime_from_unix_time_millis(ts_cell.timestamp_value) + dt = self._datetime_from_unix_time_millis( + ts_cell.timestamp_value) elif ts_cell.HasField('integer_value'): - dt = self._datetime_from_unix_time_millis(ts_cell.integer_value) + dt = self._datetime_from_unix_time_millis( + ts_cell.integer_value) logging.debug("ts_cell datetime: '%s'", dt) row.append(dt) - elif ts_col.type == riak_pb.TsColumnType.Value('BOOLEAN') and ts_cell.HasField('boolean_value'): - logging.debug("ts_cell.boolean_value: '%s'", ts_cell.boolean_value) + elif ts_col.type == riak_pb.TsColumnType.Value('BOOLEAN')\ + and ts_cell.HasField('boolean_value'): + logging.debug("ts_cell.boolean_value: '%s'", + ts_cell.boolean_value) row.append(ts_cell.boolean_value) else: row.append(None) diff --git a/riak/ts_object.py b/riak/ts_object.py index f1e4f028..ef01baff 100644 --- a/riak/ts_object.py +++ b/riak/ts_object.py @@ -1,6 +1,7 @@ from riak import RiakError from riak.table import Table + class TsObject(object): """ The TsObject holds information about Timeseries data, plus the data diff --git a/setup.py b/setup.py index d5ba1bd9..3e2b84d1 100755 --- a/setup.py +++ b/setup.py @@ -1,7 +1,6 @@ #!/usr/bin/env python import os import sys -from multiprocessing import util from setuptools import setup, find_packages from version import get_version from commands import preconfigure, configure, create_bucket_types, \ From 2c59f20487d2b4412db92d9ce447803625ab6ce7 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Tue, 27 Oct 2015 17:45:56 -0700 Subject: [PATCH 14/30] Move contributions to README like other clients. --- README.rst | 50 ++++++++++++++++++++++++++++++++++++++----- THANKS | 45 -------------------------------------- riak/tests/test_kv.py | 2 +- 3 files changed, 46 insertions(+), 51 deletions(-) delete mode 100644 THANKS diff --git a/README.rst b/README.rst index b2cb0b65..a04e86ad 100644 --- a/README.rst +++ b/README.rst @@ -247,8 +247,48 @@ To run the tests, then simply Contributors -------------------------- - - Rusty Klophaus - - Justin Sheehy - - Jay Baird - - Andy Gross - - Jon Meredith + - Andrew Thompson + - Andy Gross + - Armon Dadgar + - Brett Hazen + - Brett Hoerner + - Brian Roach + - Bryan Fink + - Daniel Lindsley + - Daniel Néri + - Daniel Reverri + - David Koblas + - Dmitry Rozhkov + - Eric Florenzano + - Eric Moritz + - Filip de Waard + - Gilles Devaux + - Greg Nelson + - Greg Stein + - Gregory Burd + - Ian Plosker + - Jayson Baird + - Jeffrey Massung + - Jon Meredith + - Josip Lisec + - Justin Sheehy + - Kevin Smith + - `Luke Bakken `_ + - Mark Erdmann + - Mark Phillips + - Mathias Meyer + - Matt Heitzenroder + - Mikhail Sobolev + - Reid Draper + - Russell Brown + - Rusty Klophaus + - Rusty Klophaus + - Scott Lystig Fritchie + - Sean Cribbs + - Shuhao Wu + - Silas Sewell + - Socrates Lee + - Soren Hansen + - Sreejith Kesavan + - Timothée Peignier + - William Kral diff --git a/THANKS b/THANKS deleted file mode 100644 index 4fccfe7f..00000000 --- a/THANKS +++ /dev/null @@ -1,45 +0,0 @@ -The following people have contributed to the Riak Python client: - -Andrew Thompson -Andy Gross -Armon Dadgar -Brett Hazen -Brett Hoerner -Brian Roach -Bryan Fink -Daniel Lindsley -Daniel Néri -Daniel Reverri -David Koblas -Dmitry Rozhkov -Eric Florenzano -Eric Moritz -Filip de Waard -Gilles Devaux -Greg Nelson -Greg Stein -Gregory Burd -Ian Plosker -Jayson Baird -Jeffrey Massung -Jon Meredith -Josip Lisec -Justin Sheehy -Kevin Smith -Mark Erdmann -Mark Phillips -Mathias Meyer -Matt Heitzenroder -Mikhail Sobolev -Reid Draper -Russell Brown -Rusty Klophaus -Scott Lystig Fritchie -Sean Cribbs -Shuhao Wu -Silas Sewell -Socrates Lee -Soren Hansen -Sreejith Kesavan -Timothée Peignier -William Kral diff --git a/riak/tests/test_kv.py b/riak/tests/test_kv.py index cc7b20c9..c9eb94dd 100644 --- a/riak/tests/test_kv.py +++ b/riak/tests/test_kv.py @@ -697,7 +697,7 @@ def test_store_binary_object_from_file(self): def test_store_binary_object_from_file_should_use_default_mimetype(self): bucket = self.client.bucket(self.bucket_name) filepath = os.path.join(os.path.dirname(os.path.abspath(__file__)), - os.pardir, os.pardir, 'THANKS') + os.pardir, os.pardir, 'README.rst') obj = bucket.new_from_file(self.key_name, filepath) obj.store() obj = bucket.get(self.key_name) From b2b15e75742ac2ad27ade007e76c853471ece9fb Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Tue, 27 Oct 2015 19:29:01 -0700 Subject: [PATCH 15/30] Small fix for test when security is disabled --- riak/tests/test_security.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/riak/tests/test_security.py b/riak/tests/test_security.py index 135bb112..9fc64e2c 100644 --- a/riak/tests/test_security.py +++ b/riak/tests/test_security.py @@ -22,7 +22,10 @@ def test_security_disabled(self): Test valid security settings without security enabled """ topts = {'timeout': 1} - client = self.create_client(credentials=SECURITY_CREDS, + # NB: can't use SECURITY_CREDS here since they won't be set + # if RUN_SECURITY is UN-set + creds = SecurityCreds(username='foo', password='bar') + client = self.create_client(credentials=creds, transport_options=topts) myBucket = client.bucket('test') val1 = "foobar" From bac106c431d2e03179e3fce568e5ee65ee28f64e Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Thu, 29 Oct 2015 10:41:45 -0700 Subject: [PATCH 16/30] Ensure format strings work correctly and make the linter happy --- riak/tests/test_timeseries.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/riak/tests/test_timeseries.py b/riak/tests/test_timeseries.py index d6f63c07..46218af6 100644 --- a/riak/tests/test_timeseries.py +++ b/riak/tests/test_timeseries.py @@ -208,16 +208,24 @@ def validate_data(self, ts_obj): self.assertIsNone(row[4]) def test_query_that_returns_no_data(self): - query = "select * from {} where time > 0 and " + \ - "time < 10 and user = 'user1'".format(table_name) + fmt = """ + select * from {table} where + time > 0 and time < 10 and user = 'user1' + """ + query = fmt.format(table=table_name) ts_obj = self.client.ts_query('GeoCheckin', query) self.assertEqual(len(ts_obj.columns), 0) self.assertEqual(len(ts_obj.rows), 0) def test_query_that_matches_some_data(self): - query = "select * from {} where time > {} and " + \ - " time < {} and user = 'user2'" \ - .format(table_name, self.tenMinsAgoMsec, self.nowMsec) + fmt = """ + select * from {table} where + time > {t1} and time < {t2} and user = 'user2' + """ + query = fmt.format( + table=table_name, + t1=self.tenMinsAgoMsec, + t2=self.nowMsec) ts_obj = self.client.ts_query('GeoCheckin', query) self.validate_data(ts_obj) From 254994412500a45a4787d4f9ad9bc58b3b5e2ec7 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Fri, 30 Oct 2015 11:20:09 -0700 Subject: [PATCH 17/30] Changes after renaming data types and fields in the PB messages --- commands.py | 25 +++++++++++++------------ riak/tests/test_security.py | 2 +- riak/tests/test_timeseries.py | 24 ++++++++++++++---------- riak/transports/pbc/codec.py | 30 +++++++++++++----------------- 4 files changed, 41 insertions(+), 40 deletions(-) diff --git a/commands.py b/commands.py index cd178e8b..d7b7923d 100644 --- a/commands.py +++ b/commands.py @@ -178,13 +178,6 @@ class create_bucket_types(bucket_type_commands, Command): class setup_timeseries(bucket_type_commands, Command): """ Creates bucket-types appropriate for timeseries. - By default this will create: - - * `GeoCheckin` with ``{"props": {"n_val": 3, - "table_def": "CREATE TABLE GeoCheckin (geohash varchar not null, - user varchar not null, time timestamp not null, - weather varchar not null, temperature float, - PRIMARY KEY((quantum(time, 15, m),user), time, user))"}}`` """ description = "create bucket-types used in timeseries tests" @@ -196,11 +189,19 @@ class setup_timeseries(bucket_type_commands, Command): _props = { 'GeoCheckin': { 'n_val': 3, - 'table_def': - 'CREATE TABLE GeoCheckin (geohash varchar not null, ' + - 'user varchar not null, time timestamp not null, ' + - 'weather varchar not null, temperature float, ' + - 'PRIMARY KEY((quantum(time, 15, m),user), time, user))'} + 'table_def': ''' + CREATE TABLE GeoCheckin ( + geohash varchar not null, + user varchar not null, + time timestamp not null, + weather varchar not null, + temperature float, + PRIMARY KEY( + (geohash, user, quantum(time, 15, m)), + geohash, user, time + ) + )''' + } } diff --git a/riak/tests/test_security.py b/riak/tests/test_security.py index 9fc64e2c..85588ee0 100644 --- a/riak/tests/test_security.py +++ b/riak/tests/test_security.py @@ -5,7 +5,7 @@ from riak.tests import RUN_SECURITY, SECURITY_USER, SECURITY_PASSWD, \ SECURITY_CACERT, SECURITY_KEY, SECURITY_CERT, SECURITY_REVOKED, \ SECURITY_CERT_USER, SECURITY_CERT_PASSWD, SECURITY_BAD_CERT, \ - SECURITY_CREDS, SECURITY_CIPHERS + SECURITY_CIPHERS from riak.security import SecurityCreds from riak.tests.base import IntegrationTestBase diff --git a/riak/tests/test_timeseries.py b/riak/tests/test_timeseries.py index 46218af6..37e9c931 100644 --- a/riak/tests/test_timeseries.py +++ b/riak/tests/test_timeseries.py @@ -69,14 +69,14 @@ def test_encode_data_for_put(self): r0 = ts_put_req.rows[0] self.assertEqual(r0.cells[0].binary_value, self.rows[0][0]) - self.assertEqual(r0.cells[1].integer_value, self.rows[0][1]) + self.assertEqual(r0.cells[1].sint64_value, self.rows[0][1]) self.assertEqual(r0.cells[2].double_value, self.rows[0][2]) self.assertEqual(r0.cells[3].timestamp_value, self.ts0ms) self.assertEqual(r0.cells[4].boolean_value, self.rows[0][4]) r1 = ts_put_req.rows[1] self.assertEqual(r1.cells[0].binary_value, self.rows[1][0]) - self.assertEqual(r1.cells[1].integer_value, self.rows[1][1]) + self.assertEqual(r1.cells[1].sint64_value, self.rows[1][1]) self.assertEqual(r1.cells[2].double_value, self.rows[1][2]) self.assertEqual(r1.cells[3].timestamp_value, self.ts1ms) self.assertEqual(r1.cells[4].boolean_value, self.rows[1][4]) @@ -89,10 +89,10 @@ def test_decode_data_from_query(self): c0.type = riak_pb.TsColumnType.Value('BINARY') c1 = tqr.columns.add() c1.name = str_to_bytes('col_integer') - c1.type = riak_pb.TsColumnType.Value('INTEGER') + c1.type = riak_pb.TsColumnType.Value('SINT64') c2 = tqr.columns.add() c2.name = str_to_bytes('col_double') - c2.type = riak_pb.TsColumnType.Value('FLOAT') + c2.type = riak_pb.TsColumnType.Value('DOUBLE') c3 = tqr.columns.add() c3.name = str_to_bytes('col_timestamp') c3.type = riak_pb.TsColumnType.Value('TIMESTAMP') @@ -104,7 +104,7 @@ def test_decode_data_from_query(self): r0c0 = r0.cells.add() r0c0.binary_value = self.rows[0][0] r0c1 = r0.cells.add() - r0c1.integer_value = self.rows[0][1] + r0c1.sint64_value = self.rows[0][1] r0c2 = r0.cells.add() r0c2.double_value = self.rows[0][2] r0c3 = r0.cells.add() @@ -116,7 +116,7 @@ def test_decode_data_from_query(self): r1c0 = r1.cells.add() r1c0.binary_value = self.rows[1][0] r1c1 = r1.cells.add() - r1c1.integer_value = self.rows[1][1] + r1c1.sint64_value = self.rows[1][1] r1c2 = r1.cells.add() r1c2.double_value = self.rows[1][2] r1c3 = r1.cells.add() @@ -135,9 +135,9 @@ def test_decode_data_from_query(self): self.assertEqual(c[0][0], 'col_binary') self.assertEqual(c[0][1], riak_pb.TsColumnType.Value('BINARY')) self.assertEqual(c[1][0], 'col_integer') - self.assertEqual(c[1][1], riak_pb.TsColumnType.Value('INTEGER')) + self.assertEqual(c[1][1], riak_pb.TsColumnType.Value('SINT64')) self.assertEqual(c[2][0], 'col_double') - self.assertEqual(c[2][1], riak_pb.TsColumnType.Value('FLOAT')) + self.assertEqual(c[2][1], riak_pb.TsColumnType.Value('DOUBLE')) self.assertEqual(c[3][0], 'col_timestamp') self.assertEqual(c[3][1], riak_pb.TsColumnType.Value('TIMESTAMP')) self.assertEqual(c[4][0], 'col_boolean') @@ -210,7 +210,9 @@ def validate_data(self, ts_obj): def test_query_that_returns_no_data(self): fmt = """ select * from {table} where - time > 0 and time < 10 and user = 'user1' + time > 0 and time < 10 and + geohash = 'hash1' and + user = 'user1' """ query = fmt.format(table=table_name) ts_obj = self.client.ts_query('GeoCheckin', query) @@ -220,7 +222,9 @@ def test_query_that_returns_no_data(self): def test_query_that_matches_some_data(self): fmt = """ select * from {table} where - time > {t1} and time < {t2} and user = 'user2' + time > {t1} and time < {t2} and + geohash = 'hash1' and + user = 'user2' """ query = fmt.format( table=table_name, diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index ac7d9122..bb5bf5f0 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -642,9 +642,9 @@ def _encode_to_ts_cell(self, cell, ts_cell): ts_cell.binary_value = str_to_bytes(cell) elif isinstance(cell, int) or isinstance(cell, long): # noqa logging.debug("cell -> int/long: '%s'", cell) - ts_cell.integer_value = cell + ts_cell.sint64_value = cell elif isinstance(cell, float): - logging.debug("cell -> float: '%s'", cell) + logging.debug("cell -> double: '%s'", cell) ts_cell.double_value = cell else: t = type(cell) @@ -732,28 +732,24 @@ def _decode_timeseries_row(self, ts_row, ts_columns): logging.debug("ts_cell.binary_value: '%s'", ts_cell.binary_value) row.append(ts_cell.binary_value) - elif ts_col.type == riak_pb.TsColumnType.Value('INTEGER')\ - and ts_cell.HasField('integer_value'): - logging.debug("ts_cell.integer_value: '%s'", - ts_cell.integer_value) - row.append(ts_cell.integer_value) - elif ts_col.type == riak_pb.TsColumnType.Value('FLOAT')\ + elif ts_col.type == riak_pb.TsColumnType.Value('SINT64')\ + and ts_cell.HasField('sint64_value'): + logging.debug("ts_cell.sint64_value: '%s'", + ts_cell.sint64_value) + row.append(ts_cell.sint64_value) + elif ts_col.type == riak_pb.TsColumnType.Value('DOUBLE')\ and ts_cell.HasField('double_value'): - value = None - if ts_cell.HasField('double_value'): - value = ts_cell.double_value - elif ts_cell.HasField('float_value'): - value = ts_cell.float_value - logging.debug("ts_cell double/float value: '%d'", value) - row.append(value) + logging.debug("ts_cell.double_value: '%d'", + ts_cell.double_value) + row.append(ts_cell.double_value) elif ts_col.type == riak_pb.TsColumnType.Value('TIMESTAMP'): dt = None if ts_cell.HasField('timestamp_value'): dt = self._datetime_from_unix_time_millis( ts_cell.timestamp_value) - elif ts_cell.HasField('integer_value'): + elif ts_cell.HasField('sint64_value'): dt = self._datetime_from_unix_time_millis( - ts_cell.integer_value) + ts_cell.sint64_value) logging.debug("ts_cell datetime: '%s'", dt) row.append(dt) elif ts_col.type == riak_pb.TsColumnType.Value('BOOLEAN')\ From d46c9404537d9c76e869ded89fdf51c56f229dfd Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Thu, 5 Nov 2015 08:45:19 -0800 Subject: [PATCH 18/30] TS integration tests are complete --- commands.py | 2 +- riak/client/operations.py | 2 +- riak/security.py | 16 ++++++------ riak/table.py | 2 +- riak/tests/test_timeseries.py | 46 ++++++++++------------------------- riak/transports/pbc/codec.py | 4 +-- 6 files changed, 25 insertions(+), 47 deletions(-) diff --git a/commands.py b/commands.py index d7b7923d..b3d41ea7 100644 --- a/commands.py +++ b/commands.py @@ -195,7 +195,7 @@ class setup_timeseries(bucket_type_commands, Command): user varchar not null, time timestamp not null, weather varchar not null, - temperature float, + temperature double, PRIMARY KEY( (geohash, user, quantum(time, 15, m)), geohash, user, time diff --git a/riak/client/operations.py b/riak/client/operations.py index 4f2d20a7..0d0c7848 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -549,7 +549,7 @@ def ts_get(self, transport, table, key): :param table: The timeseries table. :type table: string or :class:`Table ` :param key: The timeseries value's key. - :type key: list or dict + :type key: list :rtype: :class:`TsObject ` """ t = table diff --git a/riak/security.py b/riak/security.py index 542ff225..c1b36123 100644 --- a/riak/security.py +++ b/riak/security.py @@ -37,11 +37,11 @@ # For Python 2.7 and Python 3.x sslver = ssl.OPENSSL_VERSION_NUMBER # Be sure to use at least OpenSSL 1.0.1g - if sslver < OPENSSL_VERSION_101G or \ - not hasattr(ssl, 'PROTOCOL_TLSv1_2'): + tls_12 = hasattr(ssl, 'PROTOCOL_TLSv1_2') + if sslver < OPENSSL_VERSION_101G or not tls_12: verstring = ssl.OPENSSL_VERSION - msg = "Found {0} version, but expected at least OpenSSL 1.0.1g. " \ - "Security may not support TLS 1.2.".format(verstring) + msg = "{0} (>= 1.0.1g required), TLS 1.2 support: {1}" \ + .format(verstring, tls_12) warnings.warn(msg, UserWarning) if hasattr(ssl, 'PROTOCOL_TLSv1_2'): DEFAULT_TLS_VERSION = ssl.PROTOCOL_TLSv1_2 @@ -56,11 +56,11 @@ # For Python 2.6 sslver = OpenSSL.SSL.OPENSSL_VERSION_NUMBER # Be sure to use at least OpenSSL 1.0.1g - if (sslver < OPENSSL_VERSION_101G) or \ - not hasattr(OpenSSL.SSL, 'TLSv1_2_METHOD'): + tls_12 = hasattr(OpenSSL.SSL, 'TLSv1_2_METHOD') + if (sslver < OPENSSL_VERSION_101G) or tls_12: verstring = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION) - msg = "Found {0} version, but expected at least OpenSSL 1.0.1g. " \ - "Security may not support TLS 1.2.".format(verstring) + msg = "{0} (>= 1.0.1g required), TLS 1.2 support: {1}" \ + .format(verstring, tls_12) warnings.warn(msg, UserWarning) if hasattr(OpenSSL.SSL, 'TLSv1_2_METHOD'): DEFAULT_TLS_VERSION = OpenSSL.SSL.TLSv1_2_METHOD diff --git a/riak/table.py b/riak/table.py index ea30752b..27312d66 100644 --- a/riak/table.py +++ b/riak/table.py @@ -57,7 +57,7 @@ def get(self, table, key): :param table: The timeseries table. :type table: string or :class:`Table ` :param key: The timeseries value's key. - :type key: list or dict + :type key: list :rtype: :class:`TsObject ` """ return self.client.ts_get(self, table, key) diff --git a/riak/tests/test_timeseries.py b/riak/tests/test_timeseries.py index 37e9c931..6ac818bc 100644 --- a/riak/tests/test_timeseries.py +++ b/riak/tests/test_timeseries.py @@ -4,6 +4,7 @@ import platform import riak_pb +from riak import RiakError from riak.table import Table from riak.ts_object import TsObject from riak.transports.pbc.codec import RiakPbcCodec @@ -36,17 +37,15 @@ def setUp(self): [bd0, 0, 1.2, ts0, True], [bd1, 3, 4.5, ts1, False] ] - self.test_key = { - 'user': 'user2', - 'time': ts0 - } + self.test_key = [ 'hash1', 'user2', ts0 ] self.table = Table(None, 'test-table') def validate_keyreq(self, req): self.assertEqual(self.table.name, bytes_to_str(req.table)) - self.assertEqual(len(self.test_key.values()), len(req.key)) - self.assertEqual('user2', bytes_to_str(req.key[0].binary_value)) - self.assertEqual(self.ts0ms, req.key[1].timestamp_value) + self.assertEqual(len(self.test_key), len(req.key)) + self.assertEqual('hash1', bytes_to_str(req.key[0].binary_value)) + self.assertEqual('user2', bytes_to_str(req.key[1].binary_value)) + self.assertEqual(self.ts0ms, req.key[2].timestamp_value) def test_encode_data_for_get(self): req = riak_pb.TsGetReq() @@ -171,14 +170,6 @@ def setUpClass(cls): client = cls.create_client() table = client.table(table_name) - # CREATE TABLE GeoCheckin ( - # geohash varchar not null, - # user varchar not null, - # time timestamp not null, - # weather varchar not null, - # temperature float, - # PRIMARY KEY((quantum(time, 15, m), user), time, user) - # ) rows = [ ['hash1', 'user2', twentyMinsAgo, 'hurricane', 82.3], ['hash1', 'user2', fifteenMinsAgo, 'rain', 79.0], @@ -198,7 +189,7 @@ def setUpClass(cls): cls.tenMinsAgoMsec = codec._unix_time_millis(tenMinsAgo) def validate_data(self, ts_obj): - self.assertEqual(len(ts_obj.columns), 5) + # TODO self.assertEqual(len(ts_obj.columns), 5) self.assertEqual(len(ts_obj.rows), 1) row = ts_obj.rows[0] self.assertEqual(row[0], 'hash1') @@ -233,25 +224,14 @@ def test_query_that_matches_some_data(self): ts_obj = self.client.ts_query('GeoCheckin', query) self.validate_data(ts_obj) - def test_get_single_value_using_dict(self): - key = { - 'user': 'user2', - 'time': self.fiveMinsAgo - } + def test_get_single_value(self): + key = [ 'hash1', 'user2', self.fiveMinsAgo] ts_obj = self.client.ts_get('GeoCheckin', key) self.validate_data(ts_obj) - def test_get_single_value_using_array(self): - key = [self.fiveMinsAgo, 'user2'] - ts_obj = self.client.ts_get('GeoCheckin', key) - self.validate_data(ts_obj) - - def test_delete_single_value_using_dict(self): - key = { - 'user': 'user2', - 'time': self.twentyMinsAgo - } + def test_delete_single_value(self): + key = [ 'hash1', 'user2', self.twentyMinsAgo] rslt = self.client.ts_delete('GeoCheckin', key) self.assertTrue(rslt) - ts_obj = self.client.ts_get('GeoCheckin', key) - self.assertIsNone(ts_obj) + with self.assertRaises(RiakError): + self.client.ts_get('GeoCheckin', key) diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index bb5bf5f0..e343e87d 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -655,10 +655,8 @@ def _encode_timeseries_keyreq(self, table, key, req): key_vals = None if isinstance(key, list): key_vals = key - elif isinstance(key, dict): - key_vals = key.values() else: - raise ValueError("key must be a list or dict") + raise ValueError("key must be a list") req.table = str_to_bytes(table.name) for cell in key_vals: From 5f738c5b248b9a906c6d686902457b9fea32f5c1 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Tue, 10 Nov 2015 09:36:37 -0800 Subject: [PATCH 19/30] TS rpb renaming binary -> varchar, fix up tests and add some --- riak/tests/test_timeseries.py | 55 ++++++++++++++++++++++++++--------- riak/transports/pbc/codec.py | 16 +++++----- 2 files changed, 50 insertions(+), 21 deletions(-) diff --git a/riak/tests/test_timeseries.py b/riak/tests/test_timeseries.py index 6ac818bc..76dab7ed 100644 --- a/riak/tests/test_timeseries.py +++ b/riak/tests/test_timeseries.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- import datetime +import logging import os import platform import riak_pb @@ -43,8 +44,8 @@ def setUp(self): def validate_keyreq(self, req): self.assertEqual(self.table.name, bytes_to_str(req.table)) self.assertEqual(len(self.test_key), len(req.key)) - self.assertEqual('hash1', bytes_to_str(req.key[0].binary_value)) - self.assertEqual('user2', bytes_to_str(req.key[1].binary_value)) + self.assertEqual('hash1', bytes_to_str(req.key[0].varchar_value)) + self.assertEqual('user2', bytes_to_str(req.key[1].varchar_value)) self.assertEqual(self.ts0ms, req.key[2].timestamp_value) def test_encode_data_for_get(self): @@ -67,14 +68,14 @@ def test_encode_data_for_put(self): self.assertEqual(len(self.rows), len(ts_put_req.rows)) r0 = ts_put_req.rows[0] - self.assertEqual(r0.cells[0].binary_value, self.rows[0][0]) + self.assertEqual(r0.cells[0].varchar_value, self.rows[0][0]) self.assertEqual(r0.cells[1].sint64_value, self.rows[0][1]) self.assertEqual(r0.cells[2].double_value, self.rows[0][2]) self.assertEqual(r0.cells[3].timestamp_value, self.ts0ms) self.assertEqual(r0.cells[4].boolean_value, self.rows[0][4]) r1 = ts_put_req.rows[1] - self.assertEqual(r1.cells[0].binary_value, self.rows[1][0]) + self.assertEqual(r1.cells[0].varchar_value, self.rows[1][0]) self.assertEqual(r1.cells[1].sint64_value, self.rows[1][1]) self.assertEqual(r1.cells[2].double_value, self.rows[1][2]) self.assertEqual(r1.cells[3].timestamp_value, self.ts1ms) @@ -84,8 +85,8 @@ def test_decode_data_from_query(self): tqr = riak_pb.TsQueryResp() c0 = tqr.columns.add() - c0.name = str_to_bytes('col_binary') - c0.type = riak_pb.TsColumnType.Value('BINARY') + c0.name = str_to_bytes('col_varchar') + c0.type = riak_pb.TsColumnType.Value('VARCHAR') c1 = tqr.columns.add() c1.name = str_to_bytes('col_integer') c1.type = riak_pb.TsColumnType.Value('SINT64') @@ -101,7 +102,7 @@ def test_decode_data_from_query(self): r0 = tqr.rows.add() r0c0 = r0.cells.add() - r0c0.binary_value = self.rows[0][0] + r0c0.varchar_value = self.rows[0][0] r0c1 = r0.cells.add() r0c1.sint64_value = self.rows[0][1] r0c2 = r0.cells.add() @@ -113,7 +114,7 @@ def test_decode_data_from_query(self): r1 = tqr.rows.add() r1c0 = r1.cells.add() - r1c0.binary_value = self.rows[1][0] + r1c0.varchar_value = self.rows[1][0] r1c1 = r1.cells.add() r1c1.sint64_value = self.rows[1][1] r1c2 = r1.cells.add() @@ -131,8 +132,8 @@ def test_decode_data_from_query(self): self.assertEqual(len(tqr.columns), len(tsobj.columns)) c = tsobj.columns - self.assertEqual(c[0][0], 'col_binary') - self.assertEqual(c[0][1], riak_pb.TsColumnType.Value('BINARY')) + self.assertEqual(c[0][0], 'col_varchar') + self.assertEqual(c[0][1], riak_pb.TsColumnType.Value('VARCHAR')) self.assertEqual(c[1][0], 'col_integer') self.assertEqual(c[1][1], riak_pb.TsColumnType.Value('SINT64')) self.assertEqual(c[2][0], 'col_double') @@ -187,9 +188,13 @@ def setUpClass(cls): cls.fiveMinsAgo = fiveMinsAgo cls.twentyMinsAgo = twentyMinsAgo cls.tenMinsAgoMsec = codec._unix_time_millis(tenMinsAgo) + cls.twentyMinsAgoMsec = codec._unix_time_millis(twentyMinsAgo) + cls.numCols = len(rows[0]) + cls.rows = rows def validate_data(self, ts_obj): - # TODO self.assertEqual(len(ts_obj.columns), 5) + if ts_obj.columns is not None: + self.assertEqual(len(ts_obj.columns), self.numCols) self.assertEqual(len(ts_obj.rows), 1) row = ts_obj.rows[0] self.assertEqual(row[0], 'hash1') @@ -224,14 +229,38 @@ def test_query_that_matches_some_data(self): ts_obj = self.client.ts_query('GeoCheckin', query) self.validate_data(ts_obj) + def test_query_that_matches_all_data(self): + fmt = """ + select * from {table} where + time >= {t1} and time <= {t2} and + geohash = 'hash1' and + user = 'user2' + """ + query = fmt.format( + table=table_name, + t1=self.twentyMinsAgoMsec, + t2=self.nowMsec) + logging.debug("all data query: %s", query) + ts_obj = self.client.ts_query('GeoCheckin', query) + for i, want in enumerate(self.rows): + got = ts_obj.rows[i] + logging.debug("got: %s want: %s", got, want) + self.assertListEqual(got, want) + + def test_get_with_invalid_key(self): + key = [ 'hash1', 'user2' ] + with self.assertRaises(RiakError): + self.client.ts_get('GeoCheckin', key) + def test_get_single_value(self): key = [ 'hash1', 'user2', self.fiveMinsAgo] ts_obj = self.client.ts_get('GeoCheckin', key) + self.assertIsNotNone(ts_obj) self.validate_data(ts_obj) def test_delete_single_value(self): key = [ 'hash1', 'user2', self.twentyMinsAgo] rslt = self.client.ts_delete('GeoCheckin', key) self.assertTrue(rslt) - with self.assertRaises(RiakError): - self.client.ts_get('GeoCheckin', key) + ts_obj = self.client.ts_get('GeoCheckin', key) + self.assertEqual(len(ts_obj.rows), 0) diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index e343e87d..47e4abb9 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -628,8 +628,8 @@ def _encode_map_update(self, dtype, msg, op): def _encode_to_ts_cell(self, cell, ts_cell): if cell is not None: if isinstance(cell, bytes) or isinstance(cell, bytearray): - logging.debug("cell -> binary_value: '%s'", cell) - ts_cell.binary_value = cell + logging.debug("cell -> varchar_value: '%s'", cell) + ts_cell.varchar_value = cell elif isinstance(cell, datetime.datetime): ts_cell.timestamp_value = self._unix_time_millis(cell) logging.debug("cell -> timestamp: '%s', timestamp_value '%d'", @@ -639,7 +639,7 @@ def _encode_to_ts_cell(self, cell, ts_cell): ts_cell.boolean_value = cell elif isinstance(cell, str): logging.debug("cell -> str: '%s'", cell) - ts_cell.binary_value = str_to_bytes(cell) + ts_cell.varchar_value = str_to_bytes(cell) elif isinstance(cell, int) or isinstance(cell, long): # noqa logging.debug("cell -> int/long: '%s'", cell) ts_cell.sint64_value = cell @@ -725,11 +725,11 @@ def _decode_timeseries_row(self, ts_row, ts_columns): for i, ts_cell in enumerate(ts_row.cells): ts_col = ts_columns[i] logging.debug("ts_cell: '%s', ts_col: '%d'", ts_cell, ts_col.type) - if ts_col.type == riak_pb.TsColumnType.Value('BINARY')\ - and ts_cell.HasField('binary_value'): - logging.debug("ts_cell.binary_value: '%s'", - ts_cell.binary_value) - row.append(ts_cell.binary_value) + if ts_col.type == riak_pb.TsColumnType.Value('VARCHAR')\ + and ts_cell.HasField('varchar_value'): + logging.debug("ts_cell.varchar_value: '%s'", + ts_cell.varchar_value) + row.append(ts_cell.varchar_value) elif ts_col.type == riak_pb.TsColumnType.Value('SINT64')\ and ts_cell.HasField('sint64_value'): logging.debug("ts_cell.sint64_value: '%s'", From 0bcebd711e1890d46bb47b6be531b7c6df806aa5 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Mon, 23 Nov 2015 11:01:23 -0800 Subject: [PATCH 20/30] Fix tests. Timestamp is now returned in timestamp_value field --- riak/tests/test_timeseries.py | 13 ++++++++++--- riak/transports/pbc/codec.py | 12 ++++-------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/riak/tests/test_timeseries.py b/riak/tests/test_timeseries.py index 76dab7ed..ccf3b5a2 100644 --- a/riak/tests/test_timeseries.py +++ b/riak/tests/test_timeseries.py @@ -168,10 +168,12 @@ def setUpClass(cls): tenMinsAgo = fiveMinsAgo - fiveMins fifteenMinsAgo = tenMinsAgo - fiveMins twentyMinsAgo = fifteenMinsAgo - fiveMins + twentyFiveMinsAgo = twentyMinsAgo - fiveMins client = cls.create_client() table = client.table(table_name) rows = [ + ['hash1', 'user2', twentyFiveMinsAgo, 'typhoon', 90.3], ['hash1', 'user2', twentyMinsAgo, 'hurricane', 82.3], ['hash1', 'user2', fifteenMinsAgo, 'rain', 79.0], ['hash1', 'user2', fiveMinsAgo, 'wind', None], @@ -187,6 +189,7 @@ def setUpClass(cls): cls.nowMsec = codec._unix_time_millis(cls.now) cls.fiveMinsAgo = fiveMinsAgo cls.twentyMinsAgo = twentyMinsAgo + cls.twentyFiveMinsAgo = twentyFiveMinsAgo cls.tenMinsAgoMsec = codec._unix_time_millis(tenMinsAgo) cls.twentyMinsAgoMsec = codec._unix_time_millis(twentyMinsAgo) cls.numCols = len(rows[0]) @@ -229,7 +232,7 @@ def test_query_that_matches_some_data(self): ts_obj = self.client.ts_query('GeoCheckin', query) self.validate_data(ts_obj) - def test_query_that_matches_all_data(self): + def test_query_that_matches_more_data(self): fmt = """ select * from {table} where time >= {t1} and time <= {t2} and @@ -242,8 +245,12 @@ def test_query_that_matches_all_data(self): t2=self.nowMsec) logging.debug("all data query: %s", query) ts_obj = self.client.ts_query('GeoCheckin', query) + j = 0 for i, want in enumerate(self.rows): - got = ts_obj.rows[i] + if want[2] == self.twentyFiveMinsAgo: + continue + got = ts_obj.rows[j] + j += 1 logging.debug("got: %s want: %s", got, want) self.assertListEqual(got, want) @@ -259,7 +266,7 @@ def test_get_single_value(self): self.validate_data(ts_obj) def test_delete_single_value(self): - key = [ 'hash1', 'user2', self.twentyMinsAgo] + key = [ 'hash1', 'user2', self.twentyFiveMinsAgo] rslt = self.client.ts_delete('GeoCheckin', key) self.assertTrue(rslt) ts_obj = self.client.ts_get('GeoCheckin', key) diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index 47e4abb9..98668345 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -740,14 +740,10 @@ def _decode_timeseries_row(self, ts_row, ts_columns): logging.debug("ts_cell.double_value: '%d'", ts_cell.double_value) row.append(ts_cell.double_value) - elif ts_col.type == riak_pb.TsColumnType.Value('TIMESTAMP'): - dt = None - if ts_cell.HasField('timestamp_value'): - dt = self._datetime_from_unix_time_millis( - ts_cell.timestamp_value) - elif ts_cell.HasField('sint64_value'): - dt = self._datetime_from_unix_time_millis( - ts_cell.sint64_value) + elif ts_col.type == riak_pb.TsColumnType.Value('TIMESTAMP')\ + and ts_cell.HasField('timestamp_value'): + dt = self._datetime_from_unix_time_millis( + ts_cell.timestamp_value) logging.debug("ts_cell datetime: '%s'", dt) row.append(dt) elif ts_col.type == riak_pb.TsColumnType.Value('BOOLEAN')\ From 7330d75556efa40a70c67c0fd835f4a1fb605a6c Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Wed, 2 Dec 2015 23:14:07 +0000 Subject: [PATCH 21/30] PEP8 --- riak/tests/test_timeseries.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/riak/tests/test_timeseries.py b/riak/tests/test_timeseries.py index ccf3b5a2..7baf310f 100644 --- a/riak/tests/test_timeseries.py +++ b/riak/tests/test_timeseries.py @@ -38,7 +38,7 @@ def setUp(self): [bd0, 0, 1.2, ts0, True], [bd1, 3, 4.5, ts1, False] ] - self.test_key = [ 'hash1', 'user2', ts0 ] + self.test_key = ['hash1', 'user2', ts0] self.table = Table(None, 'test-table') def validate_keyreq(self, req): @@ -255,18 +255,18 @@ def test_query_that_matches_more_data(self): self.assertListEqual(got, want) def test_get_with_invalid_key(self): - key = [ 'hash1', 'user2' ] + key = ['hash1', 'user2'] with self.assertRaises(RiakError): self.client.ts_get('GeoCheckin', key) def test_get_single_value(self): - key = [ 'hash1', 'user2', self.fiveMinsAgo] + key = ['hash1', 'user2', self.fiveMinsAgo] ts_obj = self.client.ts_get('GeoCheckin', key) self.assertIsNotNone(ts_obj) self.validate_data(ts_obj) def test_delete_single_value(self): - key = [ 'hash1', 'user2', self.twentyFiveMinsAgo] + key = ['hash1', 'user2', self.twentyFiveMinsAgo] rslt = self.client.ts_delete('GeoCheckin', key) self.assertTrue(rslt) ts_obj = self.client.ts_get('GeoCheckin', key) From a48cb919612e39da6929f94d4e725970b928dfd4 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Wed, 9 Dec 2015 09:42:56 -0800 Subject: [PATCH 22/30] Add ListKeys encoding unit test --- riak/tests/test_timeseries.py | 6 +++ riak/transports/pbc/codec.py | 97 ++++++++++++++++++----------------- 2 files changed, 57 insertions(+), 46 deletions(-) diff --git a/riak/tests/test_timeseries.py b/riak/tests/test_timeseries.py index 7baf310f..9a74b6d4 100644 --- a/riak/tests/test_timeseries.py +++ b/riak/tests/test_timeseries.py @@ -81,6 +81,12 @@ def test_encode_data_for_put(self): self.assertEqual(r1.cells[3].timestamp_value, self.ts1ms) self.assertEqual(r1.cells[4].boolean_value, self.rows[1][4]) + def test_encode_data_for_listkeys(self): + req = riak_pb.TsListKeysReq() + self.c._encode_timeseries_listkeysreq(self.table, req, 1234) + self.assertEqual(self.table.name, bytes_to_str(req.table)) + self.assertEqual(1234, req.timeout) + def test_decode_data_from_query(self): tqr = riak_pb.TsQueryResp() diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index 98668345..e56adce7 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -663,24 +663,29 @@ def _encode_timeseries_keyreq(self, table, key, req): ts_cell = req.key.add() self._encode_to_ts_cell(cell, ts_cell) - def _encode_timeseries_put(self, tsobj, ts_put_req): + def _encode_timeseries_listkeysreq(self, table, req, timeout=None): + req.table = str_to_bytes(table.name) + if timeout is not None: + req.timeout = timeout + + def _encode_timeseries_put(self, tsobj, req): """ Fills an TsPutReq message with the appropriate data and metadata from a TsObject. :param tsobj: a TsObject :type tsobj: TsObject - :param ts_put_req: the protobuf message to fill - :type ts_put_req: riak_pb.TsPutReq + :param req: the protobuf message to fill + :type req: riak_pb.TsPutReq """ - ts_put_req.table = str_to_bytes(tsobj.table.name) + req.table = str_to_bytes(tsobj.table.name) if tsobj.columns: raise NotImplementedError("columns are not implemented yet") if tsobj.rows and isinstance(tsobj.rows, list): for row in tsobj.rows: - tsr = ts_put_req.rows.add() # NB: type riak_pb.TsRow + tsr = req.rows.add() # NB: type riak_pb.TsRow if not isinstance(row, list): raise ValueError("TsObject row must be a list of values") for cell in row: @@ -689,68 +694,68 @@ def _encode_timeseries_put(self, tsobj, ts_put_req): else: raise RiakError("TsObject requires a list of rows") - def _decode_timeseries(self, ts_rsp, tsobj): + def _decode_timeseries(self, resp, tsobj): """ Fills an TsObject with the appropriate data and metadata from a TsQueryResp. - :param ts_rsp: the protobuf message from which to process data - :type ts_rsp: riak_pb.TsQueryRsp or riak_pb.TsGetResp + :param resp: the protobuf message from which to process data + :type resp: riak_pb.TsQueryRsp or riak_pb.TsGetResp :param tsobj: a TsObject :type tsobj: TsObject """ if tsobj.columns is not None: - for ts_col in ts_rsp.columns: - col_name = bytes_to_str(ts_col.name) - col_type = ts_col.type + for col in resp.columns: + col_name = bytes_to_str(col.name) + col_type = col.type col = (col_name, col_type) logging.debug("column: '%s'", col) tsobj.columns.append(col) - for ts_row in ts_rsp.rows: - tsobj.rows.append(self._decode_timeseries_row(ts_row, - ts_rsp.columns)) + for row in resp.rows: + tsobj.rows.append( + self._decode_timeseries_row(row, resp.columns)) - def _decode_timeseries_row(self, ts_row, ts_columns): + def _decode_timeseries_row(self, tsrow, tscols): """ Decodes a TsRow into a list - :param ts_row: the protobuf TsRow to decode. - :type ts_row: riak_pb.TsRow - :param ts_columns: the protobuf TsColumn data to help decode. - :type ts_columns: list + :param tsrow: the protobuf TsRow to decode. + :type tsrow: riak_pb.TsRow + :param tscols: the protobuf TsColumn data to help decode. + :type tscols: list :rtype list """ row = [] - for i, ts_cell in enumerate(ts_row.cells): - ts_col = ts_columns[i] - logging.debug("ts_cell: '%s', ts_col: '%d'", ts_cell, ts_col.type) - if ts_col.type == riak_pb.TsColumnType.Value('VARCHAR')\ - and ts_cell.HasField('varchar_value'): - logging.debug("ts_cell.varchar_value: '%s'", - ts_cell.varchar_value) - row.append(ts_cell.varchar_value) - elif ts_col.type == riak_pb.TsColumnType.Value('SINT64')\ - and ts_cell.HasField('sint64_value'): - logging.debug("ts_cell.sint64_value: '%s'", - ts_cell.sint64_value) - row.append(ts_cell.sint64_value) - elif ts_col.type == riak_pb.TsColumnType.Value('DOUBLE')\ - and ts_cell.HasField('double_value'): - logging.debug("ts_cell.double_value: '%d'", - ts_cell.double_value) - row.append(ts_cell.double_value) - elif ts_col.type == riak_pb.TsColumnType.Value('TIMESTAMP')\ - and ts_cell.HasField('timestamp_value'): + for i, cell in enumerate(tsrow.cells): + col = tscols[i] + logging.debug("cell: '%s', col: '%d'", cell, col.type) + if col.type == riak_pb.TsColumnType.Value('VARCHAR')\ + and cell.HasField('varchar_value'): + logging.debug("cell.varchar_value: '%s'", + cell.varchar_value) + row.append(cell.varchar_value) + elif col.type == riak_pb.TsColumnType.Value('SINT64')\ + and cell.HasField('sint64_value'): + logging.debug("cell.sint64_value: '%s'", + cell.sint64_value) + row.append(cell.sint64_value) + elif col.type == riak_pb.TsColumnType.Value('DOUBLE')\ + and cell.HasField('double_value'): + logging.debug("cell.double_value: '%d'", + cell.double_value) + row.append(cell.double_value) + elif col.type == riak_pb.TsColumnType.Value('TIMESTAMP')\ + and cell.HasField('timestamp_value'): dt = self._datetime_from_unix_time_millis( - ts_cell.timestamp_value) - logging.debug("ts_cell datetime: '%s'", dt) + cell.timestamp_value) + logging.debug("cell datetime: '%s'", dt) row.append(dt) - elif ts_col.type == riak_pb.TsColumnType.Value('BOOLEAN')\ - and ts_cell.HasField('boolean_value'): - logging.debug("ts_cell.boolean_value: '%s'", - ts_cell.boolean_value) - row.append(ts_cell.boolean_value) + elif col.type == riak_pb.TsColumnType.Value('BOOLEAN')\ + and cell.HasField('boolean_value'): + logging.debug("cell.boolean_value: '%s'", + cell.boolean_value) + row.append(cell.boolean_value) else: row.append(None) return row From 33862c177171ae95318515f57eecffc5da377e21 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Wed, 9 Dec 2015 13:52:22 -0800 Subject: [PATCH 23/30] TS streaming list keys done and with integration test --- riak/client/operations.py | 42 +++++++++++++++++++ riak/table.py | 23 ++++++----- riak/tests/test_timeseries.py | 25 +++++++++-- riak/transports/pbc/codec.py | 71 ++++++++++++++------------------ riak/transports/pbc/stream.py | 49 ++++++++++++---------- riak/transports/pbc/transport.py | 19 ++++++++- riak/transports/transport.py | 6 +++ 7 files changed, 161 insertions(+), 74 deletions(-) diff --git a/riak/client/operations.py b/riak/client/operations.py index 0d0c7848..064c1d16 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -615,6 +615,48 @@ def ts_query(self, transport, table, query, interpolations=None): t = Table(self, table) return transport.ts_query(t, query, interpolations) + def ts_stream_keys(self, table, timeout=None): + """ + Lists all keys in a time series table via a stream. This is a + generator method which should be iterated over. + + The caller should explicitly close the returned iterator, + either using :func:`contextlib.closing` or calling ``close()`` + explicitly. Consuming the entire iterator will also close the + stream. If it does not, the associated connection might + not be returned to the pool. Example:: + + from contextlib import closing + + # Using contextlib.closing + with closing(client.ts_stream_keys(mytable)) as keys: + for key_list in keys: + do_something(key_list) + + # Explicit close() + stream = client.ts_stream_keys(mytable) + for key_list in stream: + do_something(key_list) + stream.close() + + :param table: the table from which to stream keys + :type table: Table + :param timeout: a timeout value in milliseconds + :type timeout: int + :rtype: iterator + """ + _validate_timeout(timeout) + resource = self._acquire() + transport = resource.object + stream = transport.ts_stream_keys(table, timeout) + stream.attach(resource) + try: + for keylist in stream: + if len(keylist) > 0: + yield keylist + finally: + stream.close() + @retryable def get(self, transport, robj, r=None, pr=None, timeout=None, basic_quorum=None, notfound_ok=None): diff --git a/riak/table.py b/riak/table.py index 27312d66..c477a32b 100644 --- a/riak/table.py +++ b/riak/table.py @@ -16,7 +16,6 @@ def __init__(self, client, name): :param name: The table's name :type name: string """ - if not isinstance(name, string_types): raise TypeError('Table name must be a string') @@ -50,29 +49,25 @@ def new(self, rows, columns=None): return TsObject(self._client, self, rows, columns) - def get(self, table, key): + def get(self, key): """ Gets a value from a timeseries table. - :param table: The timeseries table. - :type table: string or :class:`Table ` :param key: The timeseries value's key. :type key: list :rtype: :class:`TsObject ` """ - return self.client.ts_get(self, table, key) + return self._client.ts_get(self, key) - def delete(self, table, key): + def delete(self, key): """ Deletes a value from a timeseries table. - :param table: The timeseries table. - :type table: string or :class:`Table ` :param key: The timeseries value's key. :type key: list or dict :rtype: boolean """ - return self.client.ts_delete(self, table, key) + return self._client.ts_delete(self, key) def query(self, query, interpolations=None): """ @@ -82,4 +77,12 @@ def query(self, query, interpolations=None): :type query: string :rtype: :class:`TsObject ` """ - return self.client.ts_query(self, query, interpolations) + return self._client.ts_query(self, query, interpolations) + + def stream_keys(self, timeout=None): + """ + Streams keys from a timeseries table. + + :rtype: list + """ + return self._client.ts_stream_keys(self, timeout) diff --git a/riak/tests/test_timeseries.py b/riak/tests/test_timeseries.py index 9a74b6d4..76f4bf22 100644 --- a/riak/tests/test_timeseries.py +++ b/riak/tests/test_timeseries.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- import datetime -import logging import os import platform import riak_pb @@ -249,7 +248,6 @@ def test_query_that_matches_more_data(self): table=table_name, t1=self.twentyMinsAgoMsec, t2=self.nowMsec) - logging.debug("all data query: %s", query) ts_obj = self.client.ts_query('GeoCheckin', query) j = 0 for i, want in enumerate(self.rows): @@ -257,7 +255,6 @@ def test_query_that_matches_more_data(self): continue got = ts_obj.rows[j] j += 1 - logging.debug("got: %s want: %s", got, want) self.assertListEqual(got, want) def test_get_with_invalid_key(self): @@ -271,6 +268,28 @@ def test_get_single_value(self): self.assertIsNotNone(ts_obj) self.validate_data(ts_obj) + def test_get_single_value_via_table(self): + key = ['hash1', 'user2', self.fiveMinsAgo] + table = Table(self.client, 'GeoCheckin') + ts_obj = table.get(key) + self.assertIsNotNone(ts_obj) + self.validate_data(ts_obj) + + def test_stream_keys(self): + table = Table(self.client, 'GeoCheckin') + streamed_keys = [] + for keylist in table.stream_keys(): + self.assertNotEqual([], keylist) + streamed_keys += keylist + for key in keylist: + self.assertIsInstance(key, list) + self.assertEqual(len(key), 3) + self.assertEqual('hash1', key[0]) + self.assertEqual('user2', key[1]) + # TODO RTS-367 ENABLE + # self.assertIsInstance(key[2], datetime.datetime) + self.assertEqual(len(streamed_keys), 5) + def test_delete_single_value(self): key = ['hash1', 'user2', self.twentyFiveMinsAgo] rslt = self.client.ts_delete('GeoCheckin', key) diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index e56adce7..0a870ab7 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -1,5 +1,4 @@ import datetime -import logging import riak_pb from riak import RiakError @@ -628,23 +627,16 @@ def _encode_map_update(self, dtype, msg, op): def _encode_to_ts_cell(self, cell, ts_cell): if cell is not None: if isinstance(cell, bytes) or isinstance(cell, bytearray): - logging.debug("cell -> varchar_value: '%s'", cell) ts_cell.varchar_value = cell elif isinstance(cell, datetime.datetime): ts_cell.timestamp_value = self._unix_time_millis(cell) - logging.debug("cell -> timestamp: '%s', timestamp_value '%d'", - cell, ts_cell.timestamp_value) elif isinstance(cell, bool): - logging.debug("cell -> boolean: '%s'", cell) ts_cell.boolean_value = cell elif isinstance(cell, str): - logging.debug("cell -> str: '%s'", cell) ts_cell.varchar_value = str_to_bytes(cell) elif isinstance(cell, int) or isinstance(cell, long): # noqa - logging.debug("cell -> int/long: '%s'", cell) ts_cell.sint64_value = cell elif isinstance(cell, float): - logging.debug("cell -> double: '%s'", cell) ts_cell.double_value = cell else: t = type(cell) @@ -665,7 +657,7 @@ def _encode_timeseries_keyreq(self, table, key, req): def _encode_timeseries_listkeysreq(self, table, req, timeout=None): req.table = str_to_bytes(table.name) - if timeout is not None: + if timeout: req.timeout = timeout def _encode_timeseries_put(self, tsobj, req): @@ -709,14 +701,13 @@ def _decode_timeseries(self, resp, tsobj): col_name = bytes_to_str(col.name) col_type = col.type col = (col_name, col_type) - logging.debug("column: '%s'", col) tsobj.columns.append(col) for row in resp.rows: tsobj.rows.append( self._decode_timeseries_row(row, resp.columns)) - def _decode_timeseries_row(self, tsrow, tscols): + def _decode_timeseries_row(self, tsrow, tscols=None): """ Decodes a TsRow into a list @@ -728,34 +719,36 @@ def _decode_timeseries_row(self, tsrow, tscols): """ row = [] for i, cell in enumerate(tsrow.cells): - col = tscols[i] - logging.debug("cell: '%s', col: '%d'", cell, col.type) - if col.type == riak_pb.TsColumnType.Value('VARCHAR')\ - and cell.HasField('varchar_value'): - logging.debug("cell.varchar_value: '%s'", - cell.varchar_value) - row.append(cell.varchar_value) - elif col.type == riak_pb.TsColumnType.Value('SINT64')\ - and cell.HasField('sint64_value'): - logging.debug("cell.sint64_value: '%s'", - cell.sint64_value) - row.append(cell.sint64_value) - elif col.type == riak_pb.TsColumnType.Value('DOUBLE')\ - and cell.HasField('double_value'): - logging.debug("cell.double_value: '%d'", - cell.double_value) - row.append(cell.double_value) - elif col.type == riak_pb.TsColumnType.Value('TIMESTAMP')\ - and cell.HasField('timestamp_value'): - dt = self._datetime_from_unix_time_millis( - cell.timestamp_value) - logging.debug("cell datetime: '%s'", dt) - row.append(dt) - elif col.type == riak_pb.TsColumnType.Value('BOOLEAN')\ - and cell.HasField('boolean_value'): - logging.debug("cell.boolean_value: '%s'", - cell.boolean_value) - row.append(cell.boolean_value) + col = None + if tscols is not None: + col = tscols[i] + if cell.HasField('varchar_value'): + if col and col.type != riak_pb.TsColumnType.Value('VARCHAR'): + raise TypeError('expected VARCHAR column') + else: + row.append(cell.varchar_value) + elif cell.HasField('sint64_value'): + if col and col.type != riak_pb.TsColumnType.Value('SINT64'): + raise TypeError('expected SINT64 column') + else: + row.append(cell.sint64_value) + elif cell.HasField('double_value'): + if col and col.type != riak_pb.TsColumnType.Value('DOUBLE'): + raise TypeError('expected DOUBLE column') + else: + row.append(cell.double_value) + elif cell.HasField('timestamp_value'): + if col and col.type != riak_pb.TsColumnType.Value('TIMESTAMP'): + raise TypeError('expected TIMESTAMP column') + else: + dt = self._datetime_from_unix_time_millis( + cell.timestamp_value) + row.append(dt) + elif cell.HasField('boolean_value'): + if col and col.type != riak_pb.TsColumnType.Value('BOOLEAN'): + raise TypeError('expected BOOLEAN column') + else: + row.append(cell.boolean_value) else: row.append(None) return row diff --git a/riak/transports/pbc/stream.py b/riak/transports/pbc/stream.py index 88e7abac..6231d481 100644 --- a/riak/transports/pbc/stream.py +++ b/riak/transports/pbc/stream.py @@ -1,31 +1,14 @@ -""" -Copyright 2012 Basho Technologies, Inc. - -This file is provided to you under the Apache License, -Version 2.0 (the "License"); you may not use this file -except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -""" - - import json from riak_pb.messages import ( MSG_CODE_LIST_KEYS_RESP, MSG_CODE_MAP_RED_RESP, MSG_CODE_LIST_BUCKETS_RESP, - MSG_CODE_INDEX_RESP + MSG_CODE_INDEX_RESP, + MSG_CODE_TS_LIST_KEYS_RESP, ) from riak.util import decode_index_value, bytes_to_str from riak.client.index_page import CONTINUATION +from riak.transports.pbc.codec import RiakPbcCodec from six import PY2 @@ -55,7 +38,7 @@ def next(self): self.finished = True raise - if(self._is_done(resp)): + if self._is_done(resp): self.finished = True return resp @@ -181,3 +164,27 @@ def next(self): def __next__(self): # Python 3.x Version return self.next() + + +class RiakPbcTsKeyStream(RiakPbcStream, RiakPbcCodec): + """ + Used internally by RiakPbcTransport to implement key-list streams. + """ + + _expect = MSG_CODE_TS_LIST_KEYS_RESP + + def next(self): + response = super(RiakPbcTsKeyStream, self).next() + + if response.done and len(response.keys) is 0: + raise StopIteration + + keys = [] + for tsrow in response.keys: + keys.append(self._decode_timeseries_row(tsrow)) + + return keys + + def __next__(self): + # Python 3.x Version + return self.next() diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 4d642fce..5038310b 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -8,7 +8,8 @@ from riak.transports.pbc.stream import (RiakPbcKeyStream, RiakPbcMapredStream, RiakPbcBucketStream, - RiakPbcIndexStream) + RiakPbcIndexStream, + RiakPbcTsKeyStream) from riak.transports.pbc.codec import RiakPbcCodec from six import PY2, PY3 @@ -64,6 +65,7 @@ MSG_CODE_TS_PUT_RESP, MSG_CODE_TS_QUERY_REQ, MSG_CODE_TS_QUERY_RESP, + MSG_CODE_TS_LIST_KEYS_REQ, MSG_CODE_TS_GET_REQ, MSG_CODE_TS_GET_RESP, MSG_CODE_TS_DEL_REQ, @@ -267,6 +269,21 @@ def ts_query(self, table, query, interpolations=None): self._decode_timeseries(ts_query_resp, tsobj) return tsobj + def ts_stream_keys(self, table, timeout=None): + """ + Streams keys from a timeseries table, returning an iterator that + yields lists of keys. + """ + req = riak_pb.TsListKeysReq() + t = None + if self.client_timeouts() and timeout: + t = timeout + self._encode_timeseries_listkeysreq(table, req, t) + + self._send_msg(MSG_CODE_TS_LIST_KEYS_REQ, req) + + return RiakPbcTsKeyStream(self) + def delete(self, robj, rw=None, r=None, w=None, dw=None, pr=None, pw=None, timeout=None): req = riak_pb.RpbDelReq() diff --git a/riak/transports/transport.py b/riak/transports/transport.py index 5e2e5f3e..f9fcae6d 100644 --- a/riak/transports/transport.py +++ b/riak/transports/transport.py @@ -104,6 +104,12 @@ def ts_query(self, table, query, interpolations=None): """ raise NotImplementedError + def ts_stream_keys(self, table, timeout=None): + """ + Streams the list of keys for the table through an iterator. + """ + raise NotImplementedError + def get_buckets(self, bucket_type=None, timeout=None): """ Gets the list of buckets as strings. From e2266839776482e5c4633a8ba23c930aa489910a Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Thu, 10 Dec 2015 15:28:45 -0700 Subject: [PATCH 24/30] Tweak support for Python 3 by removing ability to have binary varchars due to Python 3's in limitation on binary encoded strings. Also tweak a few version compatibility features and clean up PEP8/pyflakes warnings. --- riak/client/operations.py | 6 +++--- riak/tests/test_timeseries.py | 15 ++++++++------- riak/transports/pbc/codec.py | 16 +++++++++------- riak/transports/pbc/transport.py | 2 +- 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/riak/client/operations.py b/riak/client/operations.py index 064c1d16..aaecae7d 100644 --- a/riak/client/operations.py +++ b/riak/client/operations.py @@ -553,7 +553,7 @@ def ts_get(self, transport, table, key): :rtype: :class:`TsObject ` """ t = table - if isinstance(t, str): + if isinstance(t, string_types): t = Table(self, table) return transport.ts_get(t, key) @@ -590,7 +590,7 @@ def ts_delete(self, transport, table, key): :rtype: boolean """ t = table - if isinstance(t, str): + if isinstance(t, string_types): t = Table(self, table) return transport.ts_delete(t, key) @@ -611,7 +611,7 @@ def ts_query(self, transport, table, query, interpolations=None): :rtype: :class:`TsObject ` """ t = table - if isinstance(t, str): + if isinstance(t, string_types): t = Table(self, table) return transport.ts_query(t, query, interpolations) diff --git a/riak/tests/test_timeseries.py b/riak/tests/test_timeseries.py index 76f4bf22..8835e21d 100644 --- a/riak/tests/test_timeseries.py +++ b/riak/tests/test_timeseries.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- import datetime -import os import platform import riak_pb @@ -19,8 +18,8 @@ table_name = 'GeoCheckin' -bd0 = os.urandom(16) -bd1 = os.urandom(16) +bd0 = '时间序列' +bd1 = 'временные ряды' fiveMins = datetime.timedelta(0, 300) ts0 = datetime.datetime(2015, 1, 1, 12, 0, 0) @@ -67,14 +66,16 @@ def test_encode_data_for_put(self): self.assertEqual(len(self.rows), len(ts_put_req.rows)) r0 = ts_put_req.rows[0] - self.assertEqual(r0.cells[0].varchar_value, self.rows[0][0]) + self.assertEqual(bytes_to_str(r0.cells[0].varchar_value), + self.rows[0][0]) self.assertEqual(r0.cells[1].sint64_value, self.rows[0][1]) self.assertEqual(r0.cells[2].double_value, self.rows[0][2]) self.assertEqual(r0.cells[3].timestamp_value, self.ts0ms) self.assertEqual(r0.cells[4].boolean_value, self.rows[0][4]) r1 = ts_put_req.rows[1] - self.assertEqual(r1.cells[0].varchar_value, self.rows[1][0]) + self.assertEqual(bytes_to_str(r1.cells[0].varchar_value), + self.rows[1][0]) self.assertEqual(r1.cells[1].sint64_value, self.rows[1][1]) self.assertEqual(r1.cells[2].double_value, self.rows[1][2]) self.assertEqual(r1.cells[3].timestamp_value, self.ts1ms) @@ -107,7 +108,7 @@ def test_decode_data_from_query(self): r0 = tqr.rows.add() r0c0 = r0.cells.add() - r0c0.varchar_value = self.rows[0][0] + r0c0.varchar_value = str_to_bytes(self.rows[0][0]) r0c1 = r0.cells.add() r0c1.sint64_value = self.rows[0][1] r0c2 = r0.cells.add() @@ -119,7 +120,7 @@ def test_decode_data_from_query(self): r1 = tqr.rows.add() r1c0 = r1.cells.add() - r1c0.varchar_value = self.rows[1][0] + r1c0.varchar_value = str_to_bytes(self.rows[1][0]) r1c1 = r1.cells.add() r1c1.sint64_value = self.rows[1][1] r1c2 = r1.cells.add() diff --git a/riak/transports/pbc/codec.py b/riak/transports/pbc/codec.py index 0a870ab7..6ec030b7 100644 --- a/riak/transports/pbc/codec.py +++ b/riak/transports/pbc/codec.py @@ -1,5 +1,6 @@ -import datetime +import logging import riak_pb +import datetime from riak import RiakError from riak.content import RiakContent @@ -626,15 +627,16 @@ def _encode_map_update(self, dtype, msg, op): def _encode_to_ts_cell(self, cell, ts_cell): if cell is not None: - if isinstance(cell, bytes) or isinstance(cell, bytearray): - ts_cell.varchar_value = cell - elif isinstance(cell, datetime.datetime): + if isinstance(cell, datetime.datetime): ts_cell.timestamp_value = self._unix_time_millis(cell) elif isinstance(cell, bool): ts_cell.boolean_value = cell - elif isinstance(cell, str): + elif isinstance(cell, string_types): + logging.debug("cell -> str: '%s'", cell) ts_cell.varchar_value = str_to_bytes(cell) - elif isinstance(cell, int) or isinstance(cell, long): # noqa + elif (isinstance(cell, int) or + (PY2 and isinstance(cell, long))): # noqa + logging.debug("cell -> int/long: '%s'", cell) ts_cell.sint64_value = cell elif isinstance(cell, float): ts_cell.double_value = cell @@ -726,7 +728,7 @@ def _decode_timeseries_row(self, tsrow, tscols=None): if col and col.type != riak_pb.TsColumnType.Value('VARCHAR'): raise TypeError('expected VARCHAR column') else: - row.append(cell.varchar_value) + row.append(bytes_to_str(cell.varchar_value)) elif cell.HasField('sint64_value'): if col and col.type != riak_pb.TsColumnType.Value('SINT64'): raise TypeError('expected SINT64 column') diff --git a/riak/transports/pbc/transport.py b/riak/transports/pbc/transport.py index 5038310b..517df987 100644 --- a/riak/transports/pbc/transport.py +++ b/riak/transports/pbc/transport.py @@ -260,7 +260,7 @@ def ts_delete(self, table, key): def ts_query(self, table, query, interpolations=None): req = riak_pb.TsQueryReq() - req.query.base = bytes_to_str(query) + req.query.base = str_to_bytes(query) msg_code, ts_query_resp = self._request(MSG_CODE_TS_QUERY_REQ, req, MSG_CODE_TS_QUERY_RESP) From 19454a2b052d6bf7417092e4d8bcaa02438cb816 Mon Sep 17 00:00:00 2001 From: Luke Bakken Date: Thu, 10 Dec 2015 14:52:13 -0800 Subject: [PATCH 25/30] Python 2.6 changes --- buildbot/Makefile | 8 ++--- buildbot/tox_cleanup.sh | 13 +++++++ riak/tests/test_timeseries.py | 68 +++++++++++++++++------------------ setup.py | 2 +- tox.ini | 6 ++++ 5 files changed, 57 insertions(+), 40 deletions(-) create mode 100755 buildbot/tox_cleanup.sh diff --git a/buildbot/Makefile b/buildbot/Makefile index 890e3984..c57f5c9e 100644 --- a/buildbot/Makefile +++ b/buildbot/Makefile @@ -26,14 +26,14 @@ test: setup test_normal test_security test_normal: @echo "Testing Riak Python Client (without security)" @../setup.py disable_security --riak-admin=${RIAK_ADMIN} - @RIAK_TEST_PROTOCOL='pbc' RUN_YZ=1 RUN_DATATYPES=1 RUN_INDEXES=1 ./tox_runner.sh .. - @RIAK_TEST_PROTOCOL='http' RUN_YZ=1 RUN_DATATYPES=1 RUN_INDEXES=1 ./tox_runner.sh .. + @RIAK_TEST_PROTOCOL='pbc' RUN_YZ=0 RUN_DATATYPES=1 RUN_INDEXES=1 ./tox_runner.sh .. + @RIAK_TEST_PROTOCOL='http' RUN_YZ=0 RUN_DATATYPES=1 RUN_INDEXES=1 ./tox_runner.sh .. test_security: @echo "Testing Riak Python Client (with security)" @../setup.py enable_security --riak-admin=${RIAK_ADMIN} - @RIAK_TEST_PROTOCOL='pbc' RUN_YZ=1 RUN_INDEXES=1 RUN_SECURITY=1 RUN_POOL=0 RUN_RESOLVE=0 ./tox_runner.sh .. - @RIAK_TEST_PROTOCOL='http' RUN_YZ=1 RUN_INDEXES=1 RUN_SECURITY=1 RUN_POOL=0 RUN_RESOLVE=0 RIAK_TEST_HTTP_PORT=18098 ./tox_runner.sh .. + @RIAK_TEST_PROTOCOL='pbc' RUN_YZ=0 RUN_INDEXES=1 RUN_SECURITY=1 RUN_POOL=0 RUN_RESOLVE=0 ./tox_runner.sh .. + @RIAK_TEST_PROTOCOL='http' RUN_YZ=0 RUN_INDEXES=1 RUN_SECURITY=1 RUN_POOL=0 RUN_RESOLVE=0 RIAK_TEST_HTTP_PORT=18098 ./tox_runner.sh .. # These are required to actually build all the Python versions: # * pip install tox diff --git a/buildbot/tox_cleanup.sh b/buildbot/tox_cleanup.sh new file mode 100755 index 00000000..bd5324c7 --- /dev/null +++ b/buildbot/tox_cleanup.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +for pbin in .tox/*/bin +do + echo $pbin + pip="$pbin/pip" + $pip uninstall riak_pb --yes + $pip uninstall riak --yes + $pip uninstall protobuf --yes + $pip uninstall python3-riak-pb --yes + $pip uninstall python3-protobuf --yes + echo ----- +done diff --git a/riak/tests/test_timeseries.py b/riak/tests/test_timeseries.py index 8835e21d..7e15b7d9 100644 --- a/riak/tests/test_timeseries.py +++ b/riak/tests/test_timeseries.py @@ -25,6 +25,26 @@ ts0 = datetime.datetime(2015, 1, 1, 12, 0, 0) ts1 = ts0 + fiveMins +now = datetime.datetime.utcfromtimestamp(144379690) +fiveMinsAgo = now - fiveMins +tenMinsAgo = fiveMinsAgo - fiveMins +fifteenMinsAgo = tenMinsAgo - fiveMins +twentyMinsAgo = fifteenMinsAgo - fiveMins +twentyFiveMinsAgo = twentyMinsAgo - fiveMins + +codec = RiakPbcCodec() +nowMsec = codec._unix_time_millis(now) +tenMinsAgoMsec = codec._unix_time_millis(tenMinsAgo) +twentyMinsAgoMsec = codec._unix_time_millis(twentyMinsAgo) + +rows = [ + ['hash1', 'user2', twentyFiveMinsAgo, 'typhoon', 90.3], + ['hash1', 'user2', twentyMinsAgo, 'hurricane', 82.3], + ['hash1', 'user2', fifteenMinsAgo, 'rain', 79.0], + ['hash1', 'user2', fiveMinsAgo, 'wind', None], + ['hash1', 'user2', now, 'snow', 20.1] +] + @unittest.skipUnless(RUN_TIMESERIES, 'RUN_TIMESERIES is 0') class TimeseriesUnitTests(unittest.TestCase): @@ -169,46 +189,23 @@ class TimeseriesTests(IntegrationTestBase, unittest.TestCase): @classmethod def setUpClass(cls): super(TimeseriesTests, cls).setUpClass() - cls.now = datetime.datetime.utcfromtimestamp(144379690) - fiveMinsAgo = cls.now - fiveMins - tenMinsAgo = fiveMinsAgo - fiveMins - fifteenMinsAgo = tenMinsAgo - fiveMins - twentyMinsAgo = fifteenMinsAgo - fiveMins - twentyFiveMinsAgo = twentyMinsAgo - fiveMins client = cls.create_client() table = client.table(table_name) - rows = [ - ['hash1', 'user2', twentyFiveMinsAgo, 'typhoon', 90.3], - ['hash1', 'user2', twentyMinsAgo, 'hurricane', 82.3], - ['hash1', 'user2', fifteenMinsAgo, 'rain', 79.0], - ['hash1', 'user2', fiveMinsAgo, 'wind', None], - ['hash1', 'user2', cls.now, 'snow', 20.1] - ] ts_obj = table.new(rows) result = ts_obj.store() if not result: raise AssertionError("expected success") client.close() - codec = RiakPbcCodec() - cls.nowMsec = codec._unix_time_millis(cls.now) - cls.fiveMinsAgo = fiveMinsAgo - cls.twentyMinsAgo = twentyMinsAgo - cls.twentyFiveMinsAgo = twentyFiveMinsAgo - cls.tenMinsAgoMsec = codec._unix_time_millis(tenMinsAgo) - cls.twentyMinsAgoMsec = codec._unix_time_millis(twentyMinsAgo) - cls.numCols = len(rows[0]) - cls.rows = rows - def validate_data(self, ts_obj): if ts_obj.columns is not None: - self.assertEqual(len(ts_obj.columns), self.numCols) + self.assertEqual(len(ts_obj.columns), 5) self.assertEqual(len(ts_obj.rows), 1) row = ts_obj.rows[0] self.assertEqual(row[0], 'hash1') self.assertEqual(row[1], 'user2') - self.assertEqual(row[2], self.fiveMinsAgo) + self.assertEqual(row[2], fiveMinsAgo) self.assertEqual(row[3], 'wind') self.assertIsNone(row[4]) @@ -233,8 +230,8 @@ def test_query_that_matches_some_data(self): """ query = fmt.format( table=table_name, - t1=self.tenMinsAgoMsec, - t2=self.nowMsec) + t1=tenMinsAgoMsec, + t2=nowMsec) ts_obj = self.client.ts_query('GeoCheckin', query) self.validate_data(ts_obj) @@ -247,12 +244,12 @@ def test_query_that_matches_more_data(self): """ query = fmt.format( table=table_name, - t1=self.twentyMinsAgoMsec, - t2=self.nowMsec) + t1=twentyMinsAgoMsec, + t2=nowMsec) ts_obj = self.client.ts_query('GeoCheckin', query) j = 0 - for i, want in enumerate(self.rows): - if want[2] == self.twentyFiveMinsAgo: + for i, want in enumerate(rows): + if want[2] == twentyFiveMinsAgo: continue got = ts_obj.rows[j] j += 1 @@ -264,13 +261,13 @@ def test_get_with_invalid_key(self): self.client.ts_get('GeoCheckin', key) def test_get_single_value(self): - key = ['hash1', 'user2', self.fiveMinsAgo] + key = ['hash1', 'user2', fiveMinsAgo] ts_obj = self.client.ts_get('GeoCheckin', key) self.assertIsNotNone(ts_obj) self.validate_data(ts_obj) def test_get_single_value_via_table(self): - key = ['hash1', 'user2', self.fiveMinsAgo] + key = ['hash1', 'user2', fiveMinsAgo] table = Table(self.client, 'GeoCheckin') ts_obj = table.get(key) self.assertIsNotNone(ts_obj) @@ -289,10 +286,11 @@ def test_stream_keys(self): self.assertEqual('user2', key[1]) # TODO RTS-367 ENABLE # self.assertIsInstance(key[2], datetime.datetime) - self.assertEqual(len(streamed_keys), 5) + keylen = len(streamed_keys) + self.assertTrue(keylen == 5 or keylen == 4) def test_delete_single_value(self): - key = ['hash1', 'user2', self.twentyFiveMinsAgo] + key = ['hash1', 'user2', twentyFiveMinsAgo] rslt = self.client.ts_delete('GeoCheckin', key) self.assertTrue(rslt) ts_obj = self.client.ts_get('GeoCheckin', key) diff --git a/setup.py b/setup.py index 3e2b84d1..d97f75b4 100755 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ os_env_pythonpath = os.environ.get('PYTHONPATH') if os_env_pythonpath is not None: for ppath in os_env_pythonpath.split(os.pathsep): - if ppath.find('riak_pb/python/lib') != -1: + if ppath.find('riak_pb/python/lib') != -1 or ppath.find('riak_pb/python3/lib') != -1: riak_pb_messages = os.path.join(ppath, 'riak_pb', 'messages.py') if os.path.exists(riak_pb_messages): riak_pb_in_pythonpath = True diff --git a/tox.ini b/tox.ini index 1bb27de4..03c15cbd 100644 --- a/tox.ini +++ b/tox.ini @@ -7,6 +7,12 @@ envlist = py26, py279, py27, py33, py34 [testenv] +basepython = + py26: python2.6 + py279: {env:HOME}/.pyenv/versions/riak-py279/bin/python2.7 + py27: python2.7 + py33: python3.3 + py34: python3.4 install_command = pip install --upgrade {packages} commands = {envpython} setup.py test deps = six From a48020d860b4225a9cf2d52e39e6c794c09839bf Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Fri, 11 Dec 2015 04:08:56 +0000 Subject: [PATCH 26/30] Update supported Python versions - Retire Python 2.6 - Swap 2.7.8 for 2.7.9 (PyOpenSSL version) - Add in Python 3.5.1 --- buildbot/Makefile | 13 +++++++++---- buildbot/tox_setup.sh | 20 ++++++++++---------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/buildbot/Makefile b/buildbot/Makefile index c57f5c9e..4be4564d 100644 --- a/buildbot/Makefile +++ b/buildbot/Makefile @@ -26,14 +26,19 @@ test: setup test_normal test_security test_normal: @echo "Testing Riak Python Client (without security)" @../setup.py disable_security --riak-admin=${RIAK_ADMIN} - @RIAK_TEST_PROTOCOL='pbc' RUN_YZ=0 RUN_DATATYPES=1 RUN_INDEXES=1 ./tox_runner.sh .. - @RIAK_TEST_PROTOCOL='http' RUN_YZ=0 RUN_DATATYPES=1 RUN_INDEXES=1 ./tox_runner.sh .. + @RIAK_TEST_PROTOCOL='pbc' RUN_YZ=1 RUN_DATATYPES=1 RUN_INDEXES=1 ./tox_runner.sh .. + @RIAK_TEST_PROTOCOL='http' RUN_YZ=1 RUN_DATATYPES=1 RUN_INDEXES=1 ./tox_runner.sh .. test_security: @echo "Testing Riak Python Client (with security)" @../setup.py enable_security --riak-admin=${RIAK_ADMIN} - @RIAK_TEST_PROTOCOL='pbc' RUN_YZ=0 RUN_INDEXES=1 RUN_SECURITY=1 RUN_POOL=0 RUN_RESOLVE=0 ./tox_runner.sh .. - @RIAK_TEST_PROTOCOL='http' RUN_YZ=0 RUN_INDEXES=1 RUN_SECURITY=1 RUN_POOL=0 RUN_RESOLVE=0 RIAK_TEST_HTTP_PORT=18098 ./tox_runner.sh .. + @RIAK_TEST_PROTOCOL='pbc' RUN_YZ=1 RUN_INDEXES=1 RUN_SECURITY=1 RUN_POOL=0 RUN_RESOLVE=0 ./tox_runner.sh .. + @RIAK_TEST_PROTOCOL='http' RUN_YZ=1 RUN_INDEXES=1 RUN_SECURITY=1 RUN_POOL=0 RUN_RESOLVE=0 RIAK_TEST_HTTP_PORT=18098 ./tox_runner.sh .. + +test_timeseries: + @echo "Testing Riak Python Client (timeseries)" + @../setup.py disable_security --riak-admin=${RIAK_ADMIN} + @RIAK_TEST_PROTOCOL='pbc' RUN_YZ=0 RUN_DATATYPES=0 RUN_INDEXES=1 RUN_TIMESERIES=1 ./tox_runner.sh .. # These are required to actually build all the Python versions: # * pip install tox diff --git a/buildbot/tox_setup.sh b/buildbot/tox_setup.sh index 1dc3f72c..05a1fe49 100755 --- a/buildbot/tox_setup.sh +++ b/buildbot/tox_setup.sh @@ -47,19 +47,19 @@ if [[ -z $(pyenv versions | grep riak_3.3.6) ]]; then VERSION_ALIAS="riak_3.3.6" pyenv install 3.3.6 pyenv virtualenv riak_3.3.6 riak-py33 fi -if [[ -z $(pyenv versions | grep riak_2.7.10) ]]; then - VERSION_ALIAS="riak_2.7.10" pyenv install 2.7.10 - pyenv virtualenv riak_2.7.10 riak-py27 +if [[ -z $(pyenv versions | grep riak_3.5.1) ]]; then + VERSION_ALIAS="riak_3.5.1" pyenv install 3.5.1 + pyenv virtualenv riak_3.5.1 riak-py35 fi -if [[ -z $(pyenv versions | grep riak_2.7.9) ]]; then - VERSION_ALIAS="riak_2.7.9" pyenv install 2.7.9 - pyenv virtualenv riak_2.7.9 riak-py279 +if [[ -z $(pyenv versions | grep riak_2.7.11) ]]; then + VERSION_ALIAS="riak_2.7.11" pyenv install 2.7.11 + pyenv virtualenv riak_2.7.11 riak-py27 fi -if [[ -z $(pyenv versions | grep riak_2.6.9) ]]; then - VERSION_ALIAS="riak_2.6.9" pyenv install 2.6.9 - pyenv virtualenv riak_2.6.9 riak-py26 +if [[ -z $(pyenv versions | grep riak_2.7.8) ]]; then + VERSION_ALIAS="riak_2.7.8" pyenv install 2.7.8 + pyenv virtualenv riak_2.7.8 riak-py278 fi -pyenv global riak-py34 riak-py33 riak-py27 riak-py279 riak-py26 +pyenv global riak-py34 riak-py33 riak-py35 riak-py27 riak-py278 pyenv versions # Now install tox From 6ae0494e5583b525619dca40dacf5057f7ebfb6f Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Fri, 11 Dec 2015 04:10:43 +0000 Subject: [PATCH 27/30] Update supported Python versions - Retire Python 2.6 - Swap 2.7.8 for 2.7.9 (PyOpenSSL version) - Add in Python 3.5.1 --- setup.py | 2 +- tox.ini | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.py b/setup.py index d97f75b4..3e2b84d1 100755 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ os_env_pythonpath = os.environ.get('PYTHONPATH') if os_env_pythonpath is not None: for ppath in os_env_pythonpath.split(os.pathsep): - if ppath.find('riak_pb/python/lib') != -1 or ppath.find('riak_pb/python3/lib') != -1: + if ppath.find('riak_pb/python/lib') != -1: riak_pb_messages = os.path.join(ppath, 'riak_pb', 'messages.py') if os.path.exists(riak_pb_messages): riak_pb_in_pythonpath = True diff --git a/tox.ini b/tox.ini index 03c15cbd..4950d459 100644 --- a/tox.ini +++ b/tox.ini @@ -4,15 +4,15 @@ # and then run "tox" from this directory. [tox] -envlist = py26, py279, py27, py33, py34 +envlist = py278, py27, py33, py34, py35 [testenv] basepython = - py26: python2.6 - py279: {env:HOME}/.pyenv/versions/riak-py279/bin/python2.7 + py278: {env:HOME}/.pyenv/versions/riak-py278/bin/python2.7 py27: python2.7 py33: python3.3 py34: python3.4 + py35: python3.5 install_command = pip install --upgrade {packages} commands = {envpython} setup.py test deps = six From fdec4412bf6b87adf95e0682bd90f309b9b6d698 Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Fri, 11 Dec 2015 04:15:57 +0000 Subject: [PATCH 28/30] Revert to 2.7+ version since 2.6 has been retired --- riak/tests/test_timeseries.py | 68 ++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/riak/tests/test_timeseries.py b/riak/tests/test_timeseries.py index 7e15b7d9..8835e21d 100644 --- a/riak/tests/test_timeseries.py +++ b/riak/tests/test_timeseries.py @@ -25,26 +25,6 @@ ts0 = datetime.datetime(2015, 1, 1, 12, 0, 0) ts1 = ts0 + fiveMins -now = datetime.datetime.utcfromtimestamp(144379690) -fiveMinsAgo = now - fiveMins -tenMinsAgo = fiveMinsAgo - fiveMins -fifteenMinsAgo = tenMinsAgo - fiveMins -twentyMinsAgo = fifteenMinsAgo - fiveMins -twentyFiveMinsAgo = twentyMinsAgo - fiveMins - -codec = RiakPbcCodec() -nowMsec = codec._unix_time_millis(now) -tenMinsAgoMsec = codec._unix_time_millis(tenMinsAgo) -twentyMinsAgoMsec = codec._unix_time_millis(twentyMinsAgo) - -rows = [ - ['hash1', 'user2', twentyFiveMinsAgo, 'typhoon', 90.3], - ['hash1', 'user2', twentyMinsAgo, 'hurricane', 82.3], - ['hash1', 'user2', fifteenMinsAgo, 'rain', 79.0], - ['hash1', 'user2', fiveMinsAgo, 'wind', None], - ['hash1', 'user2', now, 'snow', 20.1] -] - @unittest.skipUnless(RUN_TIMESERIES, 'RUN_TIMESERIES is 0') class TimeseriesUnitTests(unittest.TestCase): @@ -189,23 +169,46 @@ class TimeseriesTests(IntegrationTestBase, unittest.TestCase): @classmethod def setUpClass(cls): super(TimeseriesTests, cls).setUpClass() + cls.now = datetime.datetime.utcfromtimestamp(144379690) + fiveMinsAgo = cls.now - fiveMins + tenMinsAgo = fiveMinsAgo - fiveMins + fifteenMinsAgo = tenMinsAgo - fiveMins + twentyMinsAgo = fifteenMinsAgo - fiveMins + twentyFiveMinsAgo = twentyMinsAgo - fiveMins client = cls.create_client() table = client.table(table_name) + rows = [ + ['hash1', 'user2', twentyFiveMinsAgo, 'typhoon', 90.3], + ['hash1', 'user2', twentyMinsAgo, 'hurricane', 82.3], + ['hash1', 'user2', fifteenMinsAgo, 'rain', 79.0], + ['hash1', 'user2', fiveMinsAgo, 'wind', None], + ['hash1', 'user2', cls.now, 'snow', 20.1] + ] ts_obj = table.new(rows) result = ts_obj.store() if not result: raise AssertionError("expected success") client.close() + codec = RiakPbcCodec() + cls.nowMsec = codec._unix_time_millis(cls.now) + cls.fiveMinsAgo = fiveMinsAgo + cls.twentyMinsAgo = twentyMinsAgo + cls.twentyFiveMinsAgo = twentyFiveMinsAgo + cls.tenMinsAgoMsec = codec._unix_time_millis(tenMinsAgo) + cls.twentyMinsAgoMsec = codec._unix_time_millis(twentyMinsAgo) + cls.numCols = len(rows[0]) + cls.rows = rows + def validate_data(self, ts_obj): if ts_obj.columns is not None: - self.assertEqual(len(ts_obj.columns), 5) + self.assertEqual(len(ts_obj.columns), self.numCols) self.assertEqual(len(ts_obj.rows), 1) row = ts_obj.rows[0] self.assertEqual(row[0], 'hash1') self.assertEqual(row[1], 'user2') - self.assertEqual(row[2], fiveMinsAgo) + self.assertEqual(row[2], self.fiveMinsAgo) self.assertEqual(row[3], 'wind') self.assertIsNone(row[4]) @@ -230,8 +233,8 @@ def test_query_that_matches_some_data(self): """ query = fmt.format( table=table_name, - t1=tenMinsAgoMsec, - t2=nowMsec) + t1=self.tenMinsAgoMsec, + t2=self.nowMsec) ts_obj = self.client.ts_query('GeoCheckin', query) self.validate_data(ts_obj) @@ -244,12 +247,12 @@ def test_query_that_matches_more_data(self): """ query = fmt.format( table=table_name, - t1=twentyMinsAgoMsec, - t2=nowMsec) + t1=self.twentyMinsAgoMsec, + t2=self.nowMsec) ts_obj = self.client.ts_query('GeoCheckin', query) j = 0 - for i, want in enumerate(rows): - if want[2] == twentyFiveMinsAgo: + for i, want in enumerate(self.rows): + if want[2] == self.twentyFiveMinsAgo: continue got = ts_obj.rows[j] j += 1 @@ -261,13 +264,13 @@ def test_get_with_invalid_key(self): self.client.ts_get('GeoCheckin', key) def test_get_single_value(self): - key = ['hash1', 'user2', fiveMinsAgo] + key = ['hash1', 'user2', self.fiveMinsAgo] ts_obj = self.client.ts_get('GeoCheckin', key) self.assertIsNotNone(ts_obj) self.validate_data(ts_obj) def test_get_single_value_via_table(self): - key = ['hash1', 'user2', fiveMinsAgo] + key = ['hash1', 'user2', self.fiveMinsAgo] table = Table(self.client, 'GeoCheckin') ts_obj = table.get(key) self.assertIsNotNone(ts_obj) @@ -286,11 +289,10 @@ def test_stream_keys(self): self.assertEqual('user2', key[1]) # TODO RTS-367 ENABLE # self.assertIsInstance(key[2], datetime.datetime) - keylen = len(streamed_keys) - self.assertTrue(keylen == 5 or keylen == 4) + self.assertEqual(len(streamed_keys), 5) def test_delete_single_value(self): - key = ['hash1', 'user2', twentyFiveMinsAgo] + key = ['hash1', 'user2', self.twentyFiveMinsAgo] rslt = self.client.ts_delete('GeoCheckin', key) self.assertTrue(rslt) ts_obj = self.client.ts_get('GeoCheckin', key) From c4eaa20564593673473e1d2ea46646584cbc6b17 Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Fri, 11 Dec 2015 15:40:30 +0000 Subject: [PATCH 29/30] Simplify tox.ini for Python 2.7.8 --- tox.ini | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tox.ini b/tox.ini index 4950d459..87e74bb1 100644 --- a/tox.ini +++ b/tox.ini @@ -6,13 +6,10 @@ [tox] envlist = py278, py27, py33, py34, py35 +[testenv:py278] +basepython = {env:HOME}/.pyenv/versions/riak-py278/bin/python2.7 + [testenv] -basepython = - py278: {env:HOME}/.pyenv/versions/riak-py278/bin/python2.7 - py27: python2.7 - py33: python3.3 - py34: python3.4 - py35: python3.5 install_command = pip install --upgrade {packages} commands = {envpython} setup.py test deps = six From 57c47c02f946111038bd59d68fda2a2fb06baa43 Mon Sep 17 00:00:00 2001 From: Brett Hazen Date: Sun, 13 Dec 2015 20:27:54 +0000 Subject: [PATCH 30/30] Add Google's protobuf library as a direct prerequisite --- setup.py | 35 ++++++++++++----------------------- 1 file changed, 12 insertions(+), 23 deletions(-) diff --git a/setup.py b/setup.py index 3e2b84d1..ba3474e1 100755 --- a/setup.py +++ b/setup.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -import os -import sys +import platform from setuptools import setup, find_packages from version import get_version from commands import preconfigure, configure, create_bucket_types, \ @@ -8,33 +7,23 @@ install_requires = ['six >= 1.8.0'] requires = ['six(>=1.8.0)'] -if sys.version_info < (2, 7, 9): +if platform.python_version() < '2.7.9': install_requires.append("pyOpenSSL >= 0.14") requires.append("pyOpenSSL(>=0.14)") -riak_pb_in_pythonpath = False -os_env_pythonpath = os.environ.get('PYTHONPATH') -if os_env_pythonpath is not None: - for ppath in os_env_pythonpath.split(os.pathsep): - if ppath.find('riak_pb/python/lib') != -1: - riak_pb_messages = os.path.join(ppath, 'riak_pb', 'messages.py') - if os.path.exists(riak_pb_messages): - riak_pb_in_pythonpath = True - break - -if riak_pb_in_pythonpath: - install_requires.append("protobuf ==2.6.1") - requires.append("protobuf(==2.6.1)") +if platform.python_version() < '3.0': + install_requires.append('protobuf >=2.4.1, <2.7.0') + requires.append('protobuf(>=2.4.1,<2.7.0)') + install_requires.append("riak_pb >=2.0.0") + requires.append("riak_pb(>=2.0.0)") else: - if sys.version_info < (3, ): - install_requires.append("riak_pb >=2.0.0") - requires.append("riak_pb(>=2.0.0)") - else: - install_requires.append("python3_riak_pb >=2.0.0") - requires.append("python3_riak_pb(>=2.0.0)") + install_requires.append('python3_protobuf >=2.4.1, <2.6.0') + requires.append('python3_protobuf(>=2.4.1,<2.6.0)') + install_requires.append("python3_riak_pb >=2.0.0") + requires.append("python3_riak_pb(>=2.0.0)") tests_require = [] -if sys.version_info < (2, 7): +if platform.python_version() < '2.7.0': tests_require.append("unittest2") setup(