Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
73 commits
Select commit Hold shift + click to select a range
1d02154
added initial implementation of mutate_rows
daniel-sanche Apr 24, 2023
ab63cba
implemented mutation models
daniel-sanche Apr 24, 2023
cf9daa5
added retries to mutate_row
daniel-sanche Apr 24, 2023
1247da4
return exception group if possible
daniel-sanche Apr 24, 2023
3b3ed8c
check for idempotence
daniel-sanche Apr 24, 2023
5d20037
initial implementation for bulk_mutations
daniel-sanche Apr 24, 2023
3d322a1
include successes in bulk mutation error message
daniel-sanche Apr 24, 2023
a31232b
fixed style checks
daniel-sanche Apr 24, 2023
8da2d65
added basic system tests
daniel-sanche Apr 24, 2023
2b89d9c
added unit tests for mutate_row
daniel-sanche Apr 25, 2023
47c5985
ran blacken
daniel-sanche Apr 25, 2023
38fdcd7
improved exceptions
daniel-sanche Apr 25, 2023
504d2d8
added bulk_mutate_rows unit tests
daniel-sanche Apr 25, 2023
b16067f
ran blacken
daniel-sanche Apr 25, 2023
3ab1405
support __new___ for exceptions for python3.11+
daniel-sanche Apr 25, 2023
0a6c0c6
added exception unit tests
daniel-sanche Apr 25, 2023
ec043cf
makde exceptions tuple
daniel-sanche Apr 26, 2023
518530e
got exceptions to print consistently across versions
daniel-sanche Apr 26, 2023
9624729
added test for 311 rich traceback
daniel-sanche Apr 27, 2023
3087081
moved retryable row mutations to new file
daniel-sanche Apr 27, 2023
9df588f
use index map
daniel-sanche Apr 27, 2023
7ed8be3
added docstring
daniel-sanche Apr 27, 2023
2536cc4
added predicate check to failed mutations
daniel-sanche Apr 27, 2023
1f6875c
added _mutate_rows tests
daniel-sanche Apr 27, 2023
1ea24e6
improved client tests
daniel-sanche Apr 27, 2023
25ca2d2
refactored to loop by raising exception
daniel-sanche Apr 28, 2023
c0787db
refactored retry deadline logic into shared wrapper
daniel-sanche Apr 28, 2023
3ed5c3d
ran black
daniel-sanche Apr 28, 2023
a91fbcb
pulled in table default timeouts
daniel-sanche Apr 28, 2023
df8a058
added tests for shared deadline parsing function
daniel-sanche Apr 28, 2023
b866b57
added tests for mutation models
daniel-sanche Apr 28, 2023
54a4d43
fixed linter errors
daniel-sanche Apr 28, 2023
bd51dc4
added tests for BulkMutationsEntry
daniel-sanche Apr 28, 2023
921b05a
improved mutations documentation
daniel-sanche Apr 28, 2023
82ea61f
refactored mutate_rows logic into helper function
daniel-sanche May 2, 2023
fa42b86
implemented callbacks for mutate_rows
daniel-sanche May 2, 2023
01a16f3
made exceptions into a tuple
daniel-sanche May 5, 2023
6140acb
remove aborted from retryable errors
daniel-sanche May 22, 2023
36ba2b6
improved SetCell mutation
daniel-sanche May 22, 2023
b3c9017
fixed mutations tests
daniel-sanche May 22, 2023
cac9e2d
SetCell timestamps use millisecond precision
daniel-sanche May 22, 2023
34b051f
renamed BulkMutationsEntry to RowMutationEntry
daniel-sanche May 22, 2023
63ac35c
Merge branch 'v3' into mutate_rows
daniel-sanche May 24, 2023
a51201c
added metadata to mutate rows and bulk mutate rows
daniel-sanche May 25, 2023
a21bebf
moved _convert_retry_deadline wrapper from exceptions into _helpers
daniel-sanche May 25, 2023
4ca89d9
fixed system tests
daniel-sanche May 25, 2023
b240ee1
only handle precision adjustment when creating timestamp
daniel-sanche May 26, 2023
cb0e951
added _from_dict for mutation models
daniel-sanche May 26, 2023
a9cf385
rpc timeouts adjust when approaching operation_timeout
daniel-sanche May 26, 2023
eddc1c9
pass table instead of request dict
daniel-sanche May 26, 2023
f8b26aa
refactoring mutate rows
daniel-sanche May 27, 2023
5b80dc5
made on_terminal_state into coroutine
daniel-sanche May 27, 2023
9e5b80a
fixed style issues
daniel-sanche May 27, 2023
f7539f6
moved callback rewriting into retryable attempt
daniel-sanche May 27, 2023
e77a4fa
fixed tests
daniel-sanche May 27, 2023
4e19ed0
pop successful mutations from error dict
daniel-sanche May 30, 2023
920e4b7
removed unneeded check
daniel-sanche May 30, 2023
725f5ff
refactoring
daniel-sanche May 30, 2023
1054bc4
pass list of exceptions in callback
daniel-sanche May 30, 2023
f39a891
raise error in unexpected state
daniel-sanche May 30, 2023
1d97135
removed callback
daniel-sanche May 31, 2023
88e2bf5
refactoring mutation attempt into class
daniel-sanche May 31, 2023
a3c0166
use partial function
daniel-sanche May 31, 2023
70c35ef
renamed class
daniel-sanche May 31, 2023
e00f592
added comments
daniel-sanche Jun 1, 2023
18af78a
added tests
daniel-sanche Jun 1, 2023
23e84f5
improved helpers
daniel-sanche Jun 1, 2023
56fdf7c
refactored operation into class only
daniel-sanche Jun 5, 2023
aca31f0
restructured how remaining indices are tracked
daniel-sanche Jun 5, 2023
5a5d541
fixed tests
daniel-sanche Jun 5, 2023
afed731
added docstrings
daniel-sanche Jun 5, 2023
2396ec8
moved index deletion to end of block
daniel-sanche Jun 6, 2023
3d441a2
added comment to exception types
daniel-sanche Jun 6, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
ran blacken
  • Loading branch information
daniel-sanche committed Apr 25, 2023
commit 47c59850e4f49358e5bd7b71193d8dff0a8e9933
4 changes: 2 additions & 2 deletions google/cloud/bigtable/mutations.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,9 +65,9 @@ class DeleteRangeFromColumn(Mutation):
family: str
qualifier: bytes
# None represents 0
start_timestamp_micros: int | None=None
start_timestamp_micros: int | None = None
# None represents infinity
end_timestamp_micros: int | None=None
end_timestamp_micros: int | None = None

def _to_dict(self) -> dict[str, Any]:
timestamp_range = {}
Expand Down
13 changes: 10 additions & 3 deletions tests/system/test_system.py
Original file line number Diff line number Diff line change
Expand Up @@ -155,22 +155,29 @@ async def test_ping_and_warm_gapic(client, table):
request = {"name": table.instance_name}
await client._gapic_client.ping_and_warm(request)


@pytest.mark.asyncio
async def test_mutation_set_cell(client, table):
"""
Ensure cells can be set properly
"""
from google.cloud.bigtable.mutations import SetCell
mutation = SetCell(family=TEST_FAMILY, qualifier=b"test-qualifier", new_value=b"test-value")

mutation = SetCell(
family=TEST_FAMILY, qualifier=b"test-qualifier", new_value=b"test-value"
)
await table.mutate_row("abc", mutation)


@pytest.mark.asyncio
async def test_bulk_mutations_set_cell(client, table):
"""
Ensure cells can be set properly
"""
from google.cloud.bigtable.mutations import SetCell, BulkMutationsEntry
mutation = SetCell(family=TEST_FAMILY, qualifier=b"test-qualifier", new_value=b"test-value")

mutation = SetCell(
family=TEST_FAMILY, qualifier=b"test-qualifier", new_value=b"test-value"
)
bulk_mutation = BulkMutationsEntry(b"abc", [mutation])
await table.bulk_mutate_rows([bulk_mutation])

160 changes: 116 additions & 44 deletions tests/unit/test_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -816,97 +816,169 @@ def test_table_ctor_sync(self):
Table(client, "instance-id", "table-id")
assert e.match("Table must be created within an async event loop context.")

class TestMutateRow():

class TestMutateRow:
def _make_client(self, *args, **kwargs):
from google.cloud.bigtable.client import BigtableDataClient

return BigtableDataClient(*args, **kwargs)

@pytest.mark.asyncio
@pytest.mark.parametrize("mutation_arg", [
mutations.SetCell("family", b"qualifier", b"value"),
mutations.SetCell("family", b"qualifier", b"value", timestamp_micros=1234567890),
mutations.DeleteRangeFromColumn("family", b"qualifier"),
mutations.DeleteAllFromFamily("family"),
mutations.DeleteAllFromRow(),
[mutations.SetCell("family", b"qualifier", b"value")],
[mutations.DeleteRangeFromColumn("family", b"qualifier"), mutations.DeleteAllFromRow()],
])
@pytest.mark.parametrize(
"mutation_arg",
[
mutations.SetCell("family", b"qualifier", b"value"),
mutations.SetCell(
"family", b"qualifier", b"value", timestamp_micros=1234567890
),
mutations.DeleteRangeFromColumn("family", b"qualifier"),
mutations.DeleteAllFromFamily("family"),
mutations.DeleteAllFromRow(),
[mutations.SetCell("family", b"qualifier", b"value")],
[
mutations.DeleteRangeFromColumn("family", b"qualifier"),
mutations.DeleteAllFromRow(),
],
],
)
async def test_mutate_row(self, mutation_arg):
"""Test mutations with no errors"""
expected_per_request_timeout = 19
async with self._make_client(project="project") as client:
async with client.get_table("instance", "table") as table:
with mock.patch.object(client._gapic_client, "mutate_row") as mock_gapic:
with mock.patch.object(
client._gapic_client, "mutate_row"
) as mock_gapic:
mock_gapic.return_value = None
await table.mutate_row("row_key", mutation_arg, per_request_timeout=expected_per_request_timeout)
await table.mutate_row(
"row_key",
mutation_arg,
per_request_timeout=expected_per_request_timeout,
)
assert mock_gapic.call_count == 1
request = mock_gapic.call_args[0][0]
assert request["table_name"] == f"projects/project/instances/instance/tables/table"
assert (
request["table_name"]
== "projects/project/instances/instance/tables/table"
)
assert request["row_key"] == b"row_key"
formatted_mutations = [mutation._to_dict() for mutation in mutation_arg] if isinstance(mutation_arg, list) else [mutation_arg._to_dict()]
formatted_mutations = (
[mutation._to_dict() for mutation in mutation_arg]
if isinstance(mutation_arg, list)
else [mutation_arg._to_dict()]
)
assert request["mutations"] == formatted_mutations
found_per_request_timeout = mock_gapic.call_args[1]["timeout"]
assert found_per_request_timeout == expected_per_request_timeout


@pytest.mark.parametrize("retryable_exception", [
core_exceptions.DeadlineExceeded,
core_exceptions.ServiceUnavailable,
core_exceptions.Aborted,
])
@pytest.mark.parametrize(
"retryable_exception",
[
core_exceptions.DeadlineExceeded,
core_exceptions.ServiceUnavailable,
core_exceptions.Aborted,
],
)
@pytest.mark.asyncio
async def test_mutate_row_retryable_errors(self, retryable_exception):
from google.api_core.exceptions import DeadlineExceeded
from google.cloud.bigtable.exceptions import RetryExceptionGroup

async with self._make_client(project="project") as client:
async with client.get_table("instance", "table") as table:
with mock.patch.object(client._gapic_client, "mutate_row") as mock_gapic:
with mock.patch.object(
client._gapic_client, "mutate_row"
) as mock_gapic:
mock_gapic.side_effect = retryable_exception("mock")
with pytest.raises(DeadlineExceeded) as e:
mutation = mutations.DeleteAllFromRow()
assert mutation.is_idempotent() == True
await table.mutate_row("row_key", mutation, operation_timeout=0.05)
assert mutation.is_idempotent() is True
await table.mutate_row(
"row_key", mutation, operation_timeout=0.05
)
cause = e.value.__cause__
assert isinstance(cause, RetryExceptionGroup)
assert isinstance(cause.exceptions[0], retryable_exception)


@pytest.mark.parametrize("retryable_exception", [
core_exceptions.DeadlineExceeded,
core_exceptions.ServiceUnavailable,
core_exceptions.Aborted,
])
@pytest.mark.parametrize(
"retryable_exception",
[
core_exceptions.DeadlineExceeded,
core_exceptions.ServiceUnavailable,
core_exceptions.Aborted,
],
)
@pytest.mark.asyncio
async def test_mutate_row_non_idempotent_retryable_errors(self, retryable_exception):
async def test_mutate_row_non_idempotent_retryable_errors(
self, retryable_exception
):
"""
Non-idempotent mutations should not be retried
"""
async with self._make_client(project="project") as client:
async with client.get_table("instance", "table") as table:
with mock.patch.object(client._gapic_client, "mutate_row") as mock_gapic:
with mock.patch.object(
client._gapic_client, "mutate_row"
) as mock_gapic:
mock_gapic.side_effect = retryable_exception("mock")
with pytest.raises(retryable_exception) as e:
with pytest.raises(retryable_exception):
mutation = mutations.SetCell("family", b"qualifier", b"value")
assert mutation.is_idempotent() is False
await table.mutate_row("row_key", mutation, operation_timeout=0.2)
await table.mutate_row(
"row_key", mutation, operation_timeout=0.2
)

@pytest.mark.parametrize("non_retryable_exception", [
core_exceptions.OutOfRange,
core_exceptions.NotFound,
core_exceptions.FailedPrecondition,
RuntimeError,
ValueError,
])
@pytest.mark.parametrize(
"non_retryable_exception",
[
core_exceptions.OutOfRange,
core_exceptions.NotFound,
core_exceptions.FailedPrecondition,
RuntimeError,
ValueError,
],
)
@pytest.mark.asyncio
async def test_mutate_row_non_retryable_errors(self, non_retryable_exception):
async with self._make_client(project="project") as client:
async with client.get_table("instance", "table") as table:
with mock.patch.object(client._gapic_client, "mutate_row") as mock_gapic:
with mock.patch.object(
client._gapic_client, "mutate_row"
) as mock_gapic:
mock_gapic.side_effect = non_retryable_exception("mock")
with pytest.raises(non_retryable_exception) as e:
mutation = mutations.SetCell("family", b"qualifier", b"value", timestamp_micros=1234567890)
with pytest.raises(non_retryable_exception):
mutation = mutations.SetCell(
"family",
b"qualifier",
b"value",
timestamp_micros=1234567890,
)
assert mutation.is_idempotent() is True
await table.mutate_row("row_key", mutation, operation_timeout=0.2)
await table.mutate_row(
"row_key", mutation, operation_timeout=0.2
)


class TestBulkMutateRows:
def _make_client(self, *args, **kwargs):
from google.cloud.bigtable.client import BigtableDataClient

return BigtableDataClient(*args, **kwargs)

@pytest.mark.asyncio
def test_bulk_mutate_rows(self):
"""Test mutations with no errors"""
pass

@pytest.mark.asyncio
def test_bulk_mutate_rows_retryable_errors(self):
pass

@pytest.mark.asyncio
def test_bulk_mutate_rows_non_retryable_errors(self):
pass

@pytest.mark.asyncio
def test_bulk_mutate_rows_idempotent(self):
"""Idempotent mutations should never be retried"""
pass