Merge pull request #1290 from dhermes/bigtable-add-gc-rule

Adding primitive Bigtable garbage collection rule classes.

Author: dhermes

gcloud/_helpers.py

@@ -21,6 +21,7 @@
 import os
 from threading import local as Local
 import socket
+import sys
 
 import six
 from six.moves.http_client import HTTPConnection  # pylint: disable=F0401
@@ -260,6 +261,36 @@ def _millis_from_datetime(value):
         return _millis(value)
 
 
+def _total_seconds_backport(offset):
+    """Backport of timedelta.total_seconds() from python 2.7+.
+
+    :type offset: :class:`datetime.timedelta`
+    :param offset: A timedelta object.
+
+    :rtype: int
+    :returns: The total seconds (including microseconds) in the
+              duration.
+    """
+    seconds = offset.days * 24 * 60 * 60 + offset.seconds
+    return seconds + offset.microseconds * 1e-6
+
+
+def _total_seconds(offset):
+    """Version independent total seconds for a time delta.
+
+    :type offset: :class:`datetime.timedelta`
+    :param offset: A timedelta object.
+
+    :rtype: int
+    :returns: The total seconds (including microseconds) in the
+              duration.
+    """
+    if sys.version_info[:2] < (2, 7):  # pragma: NO COVER
+        return _total_seconds_backport(offset)
+    else:
+        return offset.total_seconds()
+
+
 def _to_bytes(value, encoding='ascii'):
     """Converts a string value to bytes, if necessary.
 

gcloud/bigtable/cluster.py

@@ -213,7 +213,7 @@ def finished(self):
         operation_name = ('operations/' + self._cluster.name +
                           '/operations/%d' % (self.op_id,))
         request_pb = operations_pb2.GetOperationRequest(name=operation_name)
-        # We expact a `._generated.operations_pb2.Operation`.
+        # We expect a `._generated.operations_pb2.Operation`.
         operation_pb = self._cluster._client._operations_stub.GetOperation(
             request_pb, self._cluster._client.timeout_seconds)
 
@@ -258,7 +258,7 @@ class Cluster(object):
 
     :type serve_nodes: int
     :param serve_nodes: (Optional) The number of nodes in the cluster.
-                        Defaults to 3.
+                        Defaults to 3 (``_DEFAULT_SERVE_NODES``).
     """
 
     def __init__(self, zone, cluster_id, client,

gcloud/bigtable/column_family.py

@@ -15,6 +15,115 @@
 """User friendly container for Google Cloud Bigtable Column Family."""
 
 
+from gcloud._helpers import _total_seconds
+from gcloud.bigtable._generated import bigtable_table_data_pb2 as data_pb2
+from gcloud.bigtable._generated import duration_pb2
+
+
+def _timedelta_to_duration_pb(timedelta_val):
+    """Convert a Python timedelta object to a duration protobuf.
+
+    .. note::
+
+        The Python timedelta has a granularity of microseconds while
+        the protobuf duration type has a duration of nanoseconds.
+
+    :type timedelta_val: :class:`datetime.timedelta`
+    :param timedelta_val: A timedelta object.
+
+    :rtype: :class:`duration_pb2.Duration`
+    :returns: A duration object equivalent to the time delta.
+    """
+    seconds_decimal = _total_seconds(timedelta_val)
+    # Truncate the parts other than the integer.
+    seconds = int(seconds_decimal)
+    if seconds_decimal < 0:
+        signed_micros = timedelta_val.microseconds - 10**6
+    else:
+        signed_micros = timedelta_val.microseconds
+    # Convert nanoseconds to microseconds.
+    nanos = 1000 * signed_micros
+    return duration_pb2.Duration(seconds=seconds, nanos=nanos)
+
+
+class GarbageCollectionRule(object):
+    """Garbage collection rule for column families within a table.
+
+    Cells in the column family (within a table) fitting the rule will be
+    deleted during garbage collection.
+
+    .. note::
+
+        This class is a do-nothing base class for all GC rules.
+
+    .. note::
+
+        A string ``gc_expression`` can also be used with API requests, but
+        that value would be superceded by a ``gc_rule``. As a result, we
+        don't support that feature and instead support via native classes.
+    """
+
+    def to_pb(self):
+        """Converts the :class:`GarbageCollectionRule` to a protobuf.
+
+        :raises: :class:`NotImplementedError <exceptions.NotImplementedError>`
+                 always since a virtual class.
+        """
+        raise NotImplementedError
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+
+class MaxVersionsGCRule(GarbageCollectionRule):
+    """Garbage collection limiting the number of versions of a cell.
+
+    :type max_num_versions: int
+    :param max_num_versions: The maximum number of versions
+    """
+
+    def __init__(self, max_num_versions):
+        self.max_num_versions = max_num_versions
+
+    def __eq__(self, other):
+        if not isinstance(other, self.__class__):
+            return False
+        return other.max_num_versions == self.max_num_versions
+
+    def to_pb(self):
+        """Converts the garbage collection rule to a protobuf.
+
+        :rtype: :class:`.data_pb2.GcRule`
+        :returns: The converted current object.
+        """
+        return data_pb2.GcRule(max_num_versions=self.max_num_versions)
+
+
+class MaxAgeGCRule(GarbageCollectionRule):
+    """Garbage collection limiting the age of a cell.
+
+    :type max_age: :class:`datetime.timedelta`
+    :param max_age: The maximum age allowed for a cell in the table.
+    """
+
+    def __init__(self, max_age):
+        self.max_age = max_age
+
+    def __eq__(self, other):
+        if not isinstance(other, self.__class__):
+            return False
+        return other.max_age == self.max_age
+
+    def to_pb(self):
+        """Converts the garbage collection rule to a protobuf.
+
+        :rtype: :class:`.data_pb2.GcRule`
+        :returns: The converted current object.
+        """
+        max_age = _timedelta_to_duration_pb(self.max_age)
+        return data_pb2.GcRule(max_age=max_age)
+
+
 class ColumnFamily(object):
     """Representation of a Google Cloud Bigtable Column Family.
 

gcloud/bigtable/table.py

@@ -41,6 +41,7 @@ class Table(object):
     We can use a :class:`Table` to:
 
     * :meth:`create` the table
+    * :meth:`rename` the table
     * :meth:`delete` the table
 
     :type table_id: str

gcloud/bigtable/test_column_family.py

@@ -16,6 +16,141 @@
 import unittest2
 
 
+class Test__timedelta_to_duration_pb(unittest2.TestCase):
+
+    def _callFUT(self, *args, **kwargs):
+        from gcloud.bigtable.column_family import _timedelta_to_duration_pb
+        return _timedelta_to_duration_pb(*args, **kwargs)
+
+    def test_it(self):
+        import datetime
+        from gcloud.bigtable._generated import duration_pb2
+
+        seconds = microseconds = 1
+        timedelta_val = datetime.timedelta(seconds=seconds,
+                                           microseconds=microseconds)
+        result = self._callFUT(timedelta_val)
+        self.assertTrue(isinstance(result, duration_pb2.Duration))
+        self.assertEqual(result.seconds, seconds)
+        self.assertEqual(result.nanos, 1000 * microseconds)
+
+    def test_with_negative_microseconds(self):
+        import datetime
+        from gcloud.bigtable._generated import duration_pb2
+
+        seconds = 1
+        microseconds = -5
+        timedelta_val = datetime.timedelta(seconds=seconds,
+                                           microseconds=microseconds)
+        result = self._callFUT(timedelta_val)
+        self.assertTrue(isinstance(result, duration_pb2.Duration))
+        self.assertEqual(result.seconds, seconds - 1)
+        self.assertEqual(result.nanos, 10**9 + 1000 * microseconds)
+
+    def test_with_negative_seconds(self):
+        import datetime
+        from gcloud.bigtable._generated import duration_pb2
+
+        seconds = -1
+        microseconds = 5
+        timedelta_val = datetime.timedelta(seconds=seconds,
+                                           microseconds=microseconds)
+        result = self._callFUT(timedelta_val)
+        self.assertTrue(isinstance(result, duration_pb2.Duration))
+        self.assertEqual(result.seconds, seconds + 1)
+        self.assertEqual(result.nanos, -(10**9 - 1000 * microseconds))
+
+
+class TestGarbageCollectionRule(unittest2.TestCase):
+
+    def _getTargetClass(self):
+        from gcloud.bigtable.column_family import GarbageCollectionRule
+        return GarbageCollectionRule
+
+    def _makeOne(self, *args, **kwargs):
+        return self._getTargetClass()(*args, **kwargs)
+
+    def test_to_pb_virtual(self):
+        gc_rule = self._makeOne()
+        self.assertRaises(NotImplementedError, gc_rule.to_pb)
+
+
+class TestMaxVersionsGCRule(unittest2.TestCase):
+
+    def _getTargetClass(self):
+        from gcloud.bigtable.column_family import MaxVersionsGCRule
+        return MaxVersionsGCRule
+
+    def _makeOne(self, *args, **kwargs):
+        return self._getTargetClass()(*args, **kwargs)
+
+    def test___eq__max_num_versions(self):
+        gc_rule1 = self._makeOne(2)
+        gc_rule2 = self._makeOne(2)
+        self.assertEqual(gc_rule1, gc_rule2)
+
+    def test___eq__type_differ(self):
+        gc_rule1 = self._makeOne(10)
+        gc_rule2 = object()
+        self.assertNotEqual(gc_rule1, gc_rule2)
+
+    def test___ne__same_value(self):
+        gc_rule1 = self._makeOne(99)
+        gc_rule2 = self._makeOne(99)
+        comparison_val = (gc_rule1 != gc_rule2)
+        self.assertFalse(comparison_val)
+
+    def test_to_pb(self):
+        from gcloud.bigtable._generated import (
+            bigtable_table_data_pb2 as data_pb2)
+        max_num_versions = 1337
+        gc_rule = self._makeOne(max_num_versions=max_num_versions)
+        pb_val = gc_rule.to_pb()
+        self.assertEqual(pb_val,
+                         data_pb2.GcRule(max_num_versions=max_num_versions))
+
+
+class TestMaxAgeGCRule(unittest2.TestCase):
+
+    def _getTargetClass(self):
+        from gcloud.bigtable.column_family import MaxAgeGCRule
+        return MaxAgeGCRule
+
+    def _makeOne(self, *args, **kwargs):
+        return self._getTargetClass()(*args, **kwargs)
+
+    def test___eq__max_age(self):
+        max_age = object()
+        gc_rule1 = self._makeOne(max_age=max_age)
+        gc_rule2 = self._makeOne(max_age=max_age)
+        self.assertEqual(gc_rule1, gc_rule2)
+
+    def test___eq__type_differ(self):
+        max_age = object()
+        gc_rule1 = self._makeOne(max_age=max_age)
+        gc_rule2 = object()
+        self.assertNotEqual(gc_rule1, gc_rule2)
+
+    def test___ne__same_value(self):
+        max_age = object()
+        gc_rule1 = self._makeOne(max_age=max_age)
+        gc_rule2 = self._makeOne(max_age=max_age)
+        comparison_val = (gc_rule1 != gc_rule2)
+        self.assertFalse(comparison_val)
+
+    def test_to_pb(self):
+        import datetime
+        from gcloud.bigtable._generated import (
+            bigtable_table_data_pb2 as data_pb2)
+        from gcloud.bigtable._generated import duration_pb2
+
+        max_age = datetime.timedelta(seconds=1)
+        duration = duration_pb2.Duration(seconds=1)
+        gc_rule = self._makeOne(max_age=max_age)
+        pb_val = gc_rule.to_pb()
+        self.assertEqual(pb_val, data_pb2.GcRule(max_age=duration))
+
+
 class TestColumnFamily(unittest2.TestCase):
 
     def _getTargetClass(self):

gcloud/test__helpers.py

@@ -354,6 +354,34 @@ def test_it(self):
         self.assertEqual(self._callFUT(NOW_MICROS), NOW)
 
 
+class Test__total_seconds_backport(unittest2.TestCase):
+
+    def _callFUT(self, *args, **kwargs):
+        from gcloud._helpers import _total_seconds_backport
+        return _total_seconds_backport(*args, **kwargs)
+
+    def test_it(self):
+        import datetime
+        offset = datetime.timedelta(seconds=3,
+                                    microseconds=140000)
+        result = self._callFUT(offset)
+        self.assertEqual(result, 3.14)
+
+
+class Test__total_seconds(unittest2.TestCase):
+
+    def _callFUT(self, *args, **kwargs):
+        from gcloud._helpers import _total_seconds
+        return _total_seconds(*args, **kwargs)
+
+    def test_it(self):
+        import datetime
+        offset = datetime.timedelta(seconds=1,
+                                    microseconds=414000)
+        result = self._callFUT(offset)
+        self.assertEqual(result, 1.414)
+
+
 class Test__to_bytes(unittest2.TestCase):
 
     def _callFUT(self, *args, **kwargs):

setup.py

@@ -14,7 +14,7 @@
 REQUIREMENTS = [
     'httplib2 >= 0.9.1',
     'oauth2client >= 1.4.6',
-    'protobuf >= 3.0.0a3',
+    'protobuf == 3.0.0a3',
     'pycrypto',
     'six',
 ]

tox.ini

@@ -10,7 +10,6 @@ commands =
 deps =
     nose
     unittest2
-    protobuf>=3.0.0a3
 setenv =
     PYTHONPATH = {toxinidir}/_testing
 covercmd =
@@ -77,7 +76,6 @@ deps =
     pep8
     pylint
     unittest2
-    protobuf==3.0.0-alpha-1
 passenv = {[testenv:system-tests]passenv}
 
 [testenv:system-tests]