From 76c899238869f5111b1986b21ab19a2c3620acdd Mon Sep 17 00:00:00 2001 From: ifaint Date: Wed, 31 Aug 2016 17:41:35 +0800 Subject: [PATCH 01/12] python 2.7 Unicode compatible. --- memcache.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/memcache.py b/memcache.py index 9823a2f..ba17616 100644 --- a/memcache.py +++ b/memcache.py @@ -148,6 +148,7 @@ class Client(threading.local): _FLAG_INTEGER = 1 << 1 _FLAG_LONG = 1 << 2 _FLAG_COMPRESSED = 1 << 3 + _FLAG_TEXT = 1 << 4 _SERVER_RETRIES = 10 # how many times to try finding a free server. @@ -968,18 +969,23 @@ def _val_to_store_info(self, val, min_compress_len): the new value itself. """ flags = 0 - if isinstance(val, six.binary_type): + # Check against the exact type, rather than using isinstance, so that + # subclasses of native types (such as markup-safe strings) are pickled + # and restored as instances of the correct class. + type_ = type(val) + if type_ == six.binary_type: pass - elif isinstance(val, six.text_type): + elif type_ == six.text_type: + flags |= Client._FLAG_TEXT val = val.encode('utf-8') - elif isinstance(val, int): + elif type_ == int: flags |= Client._FLAG_INTEGER val = '%d' % val if six.PY3: val = val.encode('ascii') # force no attempt to compress this silly string. min_compress_len = 0 - elif six.PY2 and isinstance(val, long): + elif six.PY2 and type_ == long: flags |= Client._FLAG_LONG val = str(val) if six.PY3: @@ -1266,11 +1272,10 @@ def _recv_value(self, server, flags, rlen): flags &= ~Client._FLAG_COMPRESSED if flags == 0: - # Bare string - if six.PY3: - val = buf.decode('utf8') - else: + # Bare bytes val = buf + elif flags & Client._FLAG_TEXT: + val = buf.decode('utf8') elif flags & Client._FLAG_INTEGER: val = int(buf) elif flags & Client._FLAG_LONG: From 6525f5d59498f4d2de28194d23eb58b64dfb05e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20No=C3=A9?= Date: Wed, 30 Nov 2016 16:02:40 +0100 Subject: [PATCH 02/12] Fix a str/bytes issue with Python 3. --- memcache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/memcache.py b/memcache.py index ba17616..e0ce0f3 100644 --- a/memcache.py +++ b/memcache.py @@ -581,7 +581,7 @@ def _deletetouch(self, expected, cmd, key, time=0, noreply=False): if line and line.strip() in expected: return 1 self.debuglog('%s expected %s, got: %r' - % (cmd, ' or '.join(expected), line)) + % (cmd, b' or '.join(expected), line)) except socket.error as msg: if isinstance(msg, tuple): msg = msg[1] From da1183cd7a087160ffab445202964d3502510f63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20No=C3=A9?= Date: Wed, 30 Nov 2016 16:33:27 +0100 Subject: [PATCH 03/12] Regression test: storing non-ASCII values on Python 2. --- tests/test_memcache.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_memcache.py b/tests/test_memcache.py index 321dc7b..71ed3e4 100644 --- a/tests/test_memcache.py +++ b/tests/test_memcache.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + from __future__ import print_function import unittest @@ -137,6 +139,14 @@ def test_unicode_key(self): value = self.mc.get(key) self.assertEqual(value, 5) + def test_unicode_value(self): + key = 'key' + value = six.u('Iñtërnâtiônàlizætiøn2') + + self.mc.set(key, value) + cached_value = self.mc.get(key) + self.assertEqual(value, cached_value) + def test_ignore_too_large_value(self): # NOTE: "MemCached: while expecting[...]" is normal... key = 'keyhere' From 4767c66df2a77d382b4a2426f043b1cd87d31eb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20No=C3=A9?= Date: Fri, 2 Dec 2016 12:17:00 +0100 Subject: [PATCH 04/12] Fix for touch(), with tests. --- memcache.py | 2 +- tests/test_memcache.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/memcache.py b/memcache.py index e0ce0f3..7064f4c 100644 --- a/memcache.py +++ b/memcache.py @@ -567,7 +567,7 @@ def _deletetouch(self, expected, cmd, key, time=0, noreply=False): if not server: return 0 self._statlog(cmd) - if time is not None and time != 0: + if time is not None: headers = str(time) else: headers = None diff --git a/tests/test_memcache.py b/tests/test_memcache.py index 71ed3e4..e9b5003 100644 --- a/tests/test_memcache.py +++ b/tests/test_memcache.py @@ -6,6 +6,8 @@ import six +import time + from memcache import Client, SERVER_MAX_KEY_LENGTH, SERVER_MAX_VALUE_LENGTH try: @@ -60,6 +62,34 @@ def test_delete(self): self.assertEqual(result, True) self.assertEqual(self.mc.get("long"), None) + # Delete with explicit time=0 (can re-set immediately) + self.mc.set("my_key", "my_val") + self.mc.delete("my_key", time=0) + self.assertNotEqual(self.mc.set("my_key", "my_val"), 0) + + def test_touch(self): + # Basic operation + self.mc.set("my_key", "my_val", time=1) + self.mc.touch("my_key", time=3) + time.sleep(2) + self.assertEqual(self.mc.get("my_key"), "my_val") # It's been prolonged... + time.sleep(2) + self.assertEqual(self.mc.get("my_key"), None) # But finally expires + + # Default time + self.mc.set("my_key2", "my_val", time=1) + self.mc.touch("my_key2") + time.sleep(2) + self.assertEqual(self.mc.get("my_key2"), "my_val") + # TODO: test it stays forever + + # Return values + self.mc.set("my_key3", "my_val") + result = self.mc.touch("my_key3") + self.assertNotEqual(result, 0) # Returns nonzero on success + result = self.mc.touch("inexisting_key") + self.assertEqual(result, 0) + def test_get_multi(self): self.check_setget("gm_a_string", "some random string") self.check_setget("gm_an_integer", 42) From 0143a607142db948c7af92c4da772b91f6637424 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20No=C3=A9?= Date: Wed, 14 Dec 2016 10:09:04 +0100 Subject: [PATCH 05/12] Fix for #79 (thanks dzhuang), with test (thanks Timgraham). --- memcache.py | 23 ++++++++++++++--------- tests/test_memcache.py | 10 ++++++++++ 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/memcache.py b/memcache.py index 9823a2f..33ed7c5 100644 --- a/memcache.py +++ b/memcache.py @@ -148,6 +148,7 @@ class Client(threading.local): _FLAG_INTEGER = 1 << 1 _FLAG_LONG = 1 << 2 _FLAG_COMPRESSED = 1 << 3 + _FLAG_TEXT = 1 << 4 _SERVER_RETRIES = 10 # how many times to try finding a free server. @@ -968,18 +969,23 @@ def _val_to_store_info(self, val, min_compress_len): the new value itself. """ flags = 0 - if isinstance(val, six.binary_type): + # Check against the exact type, rather than using isinstance, so that + # subclasses of native types (such as markup-safe strings) are pickled + # and restored as instances of the correct class. + type_ = type(val) + if type_ == six.binary_type: pass - elif isinstance(val, six.text_type): + elif type_ == six.text_type: + flags |= Client._FLAG_TEXT val = val.encode('utf-8') - elif isinstance(val, int): + elif type_ == int: flags |= Client._FLAG_INTEGER val = '%d' % val if six.PY3: val = val.encode('ascii') # force no attempt to compress this silly string. min_compress_len = 0 - elif six.PY2 and isinstance(val, long): + elif six.PY2 and type_ == long: flags |= Client._FLAG_LONG val = str(val) if six.PY3: @@ -1266,11 +1272,10 @@ def _recv_value(self, server, flags, rlen): flags &= ~Client._FLAG_COMPRESSED if flags == 0: - # Bare string - if six.PY3: - val = buf.decode('utf8') - else: + # Bare bytes val = buf + elif flags & Client._FLAG_TEXT: + val = buf.decode('utf8') elif flags & Client._FLAG_INTEGER: val = int(buf) elif flags & Client._FLAG_LONG: @@ -1523,4 +1528,4 @@ def _doctest(): sys.exit(1) -# vim: ts=4 sw=4 et : +# vim: ts=4 sw=4 et : \ No newline at end of file diff --git a/tests/test_memcache.py b/tests/test_memcache.py index 321dc7b..71ed3e4 100644 --- a/tests/test_memcache.py +++ b/tests/test_memcache.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + from __future__ import print_function import unittest @@ -137,6 +139,14 @@ def test_unicode_key(self): value = self.mc.get(key) self.assertEqual(value, 5) + def test_unicode_value(self): + key = 'key' + value = six.u('Iñtërnâtiônàlizætiøn2') + + self.mc.set(key, value) + cached_value = self.mc.get(key) + self.assertEqual(value, cached_value) + def test_ignore_too_large_value(self): # NOTE: "MemCached: while expecting[...]" is normal... key = 'keyhere' From a3be492d34d9d587ab29de9cf2f9aab7b8b59eda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20No=C3=A9?= Date: Wed, 14 Dec 2016 14:25:41 +0100 Subject: [PATCH 06/12] Added missing newline at end of file. --- memcache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/memcache.py b/memcache.py index 33ed7c5..ba17616 100644 --- a/memcache.py +++ b/memcache.py @@ -1528,4 +1528,4 @@ def _doctest(): sys.exit(1) -# vim: ts=4 sw=4 et : \ No newline at end of file +# vim: ts=4 sw=4 et : From 2cff22a14a50c1f738722075795e7f8befbfeddb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20No=C3=A9?= Date: Wed, 14 Dec 2016 14:40:53 +0100 Subject: [PATCH 07/12] Added tests to ensure #80 is fixed. --- tests/test_memcache.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/test_memcache.py b/tests/test_memcache.py index 71ed3e4..66f7007 100644 --- a/tests/test_memcache.py +++ b/tests/test_memcache.py @@ -147,6 +147,30 @@ def test_unicode_value(self): cached_value = self.mc.get(key) self.assertEqual(value, cached_value) + def test_binary_string(self): + # Binary strings should be cacheable + from zlib import compress, decompress + value = 'value_to_be_compressed' + compressed_value = compress(value.encode()) + + # Test set + self.mc.set('binary1', compressed_value) + compressed_result = self.mc.get('binary1') + self.assertEqual(compressed_value, compressed_result) + self.assertEqual(value, decompress(compressed_result).decode()) + + # Test add + self.mc.add('binary1-add', compressed_value) + compressed_result = self.mc.get('binary1-add') + self.assertEqual(compressed_value, compressed_result) + self.assertEqual(value, decompress(compressed_result).decode()) + + # Test set_many + self.mc.set_multi({'binary1-set_many': compressed_value}) + compressed_result = self.mc.get('binary1-set_many') + self.assertEqual(compressed_value, compressed_result) + self.assertEqual(value, decompress(compressed_result).decode()) + def test_ignore_too_large_value(self): # NOTE: "MemCached: while expecting[...]" is normal... key = 'keyhere' From 1461ea3241dcc3a9d6134a2f7a51064e8788be35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20No=C3=A9?= Date: Wed, 14 Dec 2016 14:43:24 +0100 Subject: [PATCH 08/12] Improved style --- memcache.py | 2 +- tests/test_memcache.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/memcache.py b/memcache.py index ba17616..909daec 100644 --- a/memcache.py +++ b/memcache.py @@ -1273,7 +1273,7 @@ def _recv_value(self, server, flags, rlen): if flags == 0: # Bare bytes - val = buf + val = buf elif flags & Client._FLAG_TEXT: val = buf.decode('utf8') elif flags & Client._FLAG_INTEGER: diff --git a/tests/test_memcache.py b/tests/test_memcache.py index 66f7007..afac9d6 100644 --- a/tests/test_memcache.py +++ b/tests/test_memcache.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - from __future__ import print_function import unittest @@ -142,7 +141,6 @@ def test_unicode_key(self): def test_unicode_value(self): key = 'key' value = six.u('Iñtërnâtiônàlizætiøn2') - self.mc.set(key, value) cached_value = self.mc.get(key) self.assertEqual(value, cached_value) From 4fdb82ffec53e6499f53a5c588cdad37434bda69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20No=C3=A9?= Date: Tue, 20 Dec 2016 11:25:12 +0100 Subject: [PATCH 09/12] Improved style --- memcache.py | 18 +++++++++--------- tests/test_memcache.py | 16 ++++++---------- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/memcache.py b/memcache.py index 41fc8aa..2097f58 100644 --- a/memcache.py +++ b/memcache.py @@ -244,9 +244,9 @@ def __init__(self, servers, debug=0, pickleProtocol=0, def _encode_key(self, key): if isinstance(key, tuple): if isinstance(key[1], six.text_type): - return (key[0], key[1].encode('utf8')) + return (key[0], key[1].encode('utf-8')) elif isinstance(key, six.text_type): - return key.encode('utf8') + return key.encode('utf-8') return key def _encode_cmd(self, cmd, key, headers, noreply, *args): @@ -793,7 +793,7 @@ def _map_and_prefix_keys(self, key_iterable, key_prefix): # set_multi supports int / long keys. key = str(key) if six.PY3: - key = key.encode('utf8') + key = key.encode('utf-8') bytes_orig_key = key # Gotta pre-mangle key before hashing to a @@ -808,7 +808,7 @@ def _map_and_prefix_keys(self, key_iterable, key_prefix): # set_multi supports int / long keys. key = str(key) if six.PY3: - key = key.encode('utf8') + key = key.encode('utf-8') bytes_orig_key = key server, key = self._get_server(key_prefix + key) @@ -1261,7 +1261,7 @@ def _recv_value(self, server, flags, rlen): # Bare bytes val = buf elif flags & Client._FLAG_TEXT: - val = buf.decode('utf8') + val = buf.decode('utf-8') elif flags & Client._FLAG_INTEGER: val = int(buf) elif flags & Client._FLAG_LONG: @@ -1422,13 +1422,13 @@ def close_socket(self): def send_cmd(self, cmd): if isinstance(cmd, six.text_type): - cmd = cmd.encode('utf8') + cmd = cmd.encode('utf-8') self.socket.sendall(cmd + b'\r\n') def send_cmds(self, cmds): """cmds already has trailing \r\n's applied.""" if isinstance(cmds, six.text_type): - cmds = cmds.encode('utf8') + cmds = cmds.encode('utf-8') self.socket.sendall(cmds) def readline(self, raise_exception=False): @@ -1464,8 +1464,8 @@ def expect(self, text, raise_exception=False): line = self.readline(raise_exception) if self.debug and line != text: if six.PY3: - text = text.decode('utf8') - log_line = line.decode('utf8', 'replace') + text = text.decode('utf-8') + log_line = line.decode('utf-8', 'replace') else: log_line = line self.debuglog("while expecting %r, got unexpected response %r" diff --git a/tests/test_memcache.py b/tests/test_memcache.py index 4b8aaea..9d8b4e9 100644 --- a/tests/test_memcache.py +++ b/tests/test_memcache.py @@ -7,6 +7,8 @@ import time +import zlib + from memcache import Client, SERVER_MAX_KEY_LENGTH, SERVER_MAX_VALUE_LENGTH # noqa: H301 @@ -160,34 +162,28 @@ def test_unicode_key(self): def test_unicode_value(self): key = 'key' value = six.u('Iñtërnâtiônàlizætiøn2') - self.mc.set(key, value) cached_value = self.mc.get(key) self.assertEqual(value, cached_value) def test_binary_string(self): - # Binary strings should be cacheable - from zlib import compress, decompress value = 'value_to_be_compressed' - compressed_value = compress(value.encode()) + compressed_value = zlib.compress(value.encode()) - # Test set self.mc.set('binary1', compressed_value) compressed_result = self.mc.get('binary1') self.assertEqual(compressed_value, compressed_result) - self.assertEqual(value, decompress(compressed_result).decode()) + self.assertEqual(value, zlib.decompress(compressed_result).decode()) - # Test add self.mc.add('binary1-add', compressed_value) compressed_result = self.mc.get('binary1-add') self.assertEqual(compressed_value, compressed_result) - self.assertEqual(value, decompress(compressed_result).decode()) + self.assertEqual(value, zlib.decompress(compressed_result).decode()) - # Test set_many self.mc.set_multi({'binary1-set_many': compressed_value}) compressed_result = self.mc.get('binary1-set_many') self.assertEqual(compressed_value, compressed_result) - self.assertEqual(value, decompress(compressed_result).decode()) + self.assertEqual(value, zlib.decompress(compressed_result).decode()) def test_ignore_too_large_value(self): # NOTE: "MemCached: while expecting[...]" is normal... From f6c7fe3a1626ff1a7b708ebffcd8c434d40bb5c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20No=C3=A9?= Date: Tue, 20 Dec 2016 11:29:20 +0100 Subject: [PATCH 10/12] Removed old TODO / fixes flake8 --- tests/test_memcache.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_memcache.py b/tests/test_memcache.py index 9d8b4e9..d3dbc52 100644 --- a/tests/test_memcache.py +++ b/tests/test_memcache.py @@ -71,7 +71,6 @@ def test_touch(self): self.mc.touch("my_key2") time.sleep(2) self.assertEqual(self.mc.get("my_key2"), "my_val") - # TODO: test it stays forever # Return values self.mc.set("my_key3", "my_val") From 7475d96b3cfe1fdb8f5d8670f521fc60c8063828 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20No=C3=A9?= Date: Tue, 18 Jul 2017 12:53:46 +0200 Subject: [PATCH 11/12] Removed unrelated cleanups. --- memcache.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/memcache.py b/memcache.py index 2097f58..5e841a7 100644 --- a/memcache.py +++ b/memcache.py @@ -244,9 +244,9 @@ def __init__(self, servers, debug=0, pickleProtocol=0, def _encode_key(self, key): if isinstance(key, tuple): if isinstance(key[1], six.text_type): - return (key[0], key[1].encode('utf-8')) + return (key[0], key[1].encode('utf8')) elif isinstance(key, six.text_type): - return key.encode('utf-8') + return key.encode('utf8') return key def _encode_cmd(self, cmd, key, headers, noreply, *args): @@ -793,7 +793,7 @@ def _map_and_prefix_keys(self, key_iterable, key_prefix): # set_multi supports int / long keys. key = str(key) if six.PY3: - key = key.encode('utf-8') + key = key.encode('utf8') bytes_orig_key = key # Gotta pre-mangle key before hashing to a @@ -808,7 +808,7 @@ def _map_and_prefix_keys(self, key_iterable, key_prefix): # set_multi supports int / long keys. key = str(key) if six.PY3: - key = key.encode('utf-8') + key = key.encode('utf8') bytes_orig_key = key server, key = self._get_server(key_prefix + key) @@ -1422,13 +1422,13 @@ def close_socket(self): def send_cmd(self, cmd): if isinstance(cmd, six.text_type): - cmd = cmd.encode('utf-8') + cmd = cmd.encode('utf8') self.socket.sendall(cmd + b'\r\n') def send_cmds(self, cmds): """cmds already has trailing \r\n's applied.""" if isinstance(cmds, six.text_type): - cmds = cmds.encode('utf-8') + cmds = cmds.encode('utf8') self.socket.sendall(cmds) def readline(self, raise_exception=False): @@ -1464,8 +1464,8 @@ def expect(self, text, raise_exception=False): line = self.readline(raise_exception) if self.debug and line != text: if six.PY3: - text = text.decode('utf-8') - log_line = line.decode('utf-8', 'replace') + text = text.decode('utf8') + log_line = line.decode('utf8', 'replace') else: log_line = line self.debuglog("while expecting %r, got unexpected response %r" From 912bb178e68c4d098079b677b8ca00caefd5b296 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicolas=20No=C3=A9?= Date: Tue, 18 Jul 2017 13:09:47 +0200 Subject: [PATCH 12/12] Removed unrelated fix (touch() method). --- memcache.py | 2 +- tests/test_memcache.py | 29 ----------------------------- 2 files changed, 1 insertion(+), 30 deletions(-) diff --git a/memcache.py b/memcache.py index 5e841a7..30accb8 100644 --- a/memcache.py +++ b/memcache.py @@ -553,7 +553,7 @@ def _deletetouch(self, expected, cmd, key, time=0, noreply=False): if not server: return 0 self._statlog(cmd) - if time is not None: + if time is not None and time != 0: headers = str(time) else: headers = None diff --git a/tests/test_memcache.py b/tests/test_memcache.py index d3dbc52..51b431c 100644 --- a/tests/test_memcache.py +++ b/tests/test_memcache.py @@ -5,8 +5,6 @@ import six -import time - import zlib from memcache import Client, SERVER_MAX_KEY_LENGTH, SERVER_MAX_VALUE_LENGTH # noqa: H301 @@ -52,33 +50,6 @@ def test_delete(self): self.assertEqual(result, True) self.assertEqual(self.mc.get("long"), None) - # Delete with explicit time=0 (can re-set immediately) - self.mc.set("my_key", "my_val") - self.mc.delete("my_key", time=0) - self.assertNotEqual(self.mc.set("my_key", "my_val"), 0) - - def test_touch(self): - # Basic operation - self.mc.set("my_key", "my_val", time=1) - self.mc.touch("my_key", time=3) - time.sleep(2) - self.assertEqual(self.mc.get("my_key"), "my_val") # It's been prolonged... - time.sleep(2) - self.assertEqual(self.mc.get("my_key"), None) # But finally expires - - # Default time - self.mc.set("my_key2", "my_val", time=1) - self.mc.touch("my_key2") - time.sleep(2) - self.assertEqual(self.mc.get("my_key2"), "my_val") - - # Return values - self.mc.set("my_key3", "my_val") - result = self.mc.touch("my_key3") - self.assertNotEqual(result, 0) # Returns nonzero on success - result = self.mc.touch("inexisting_key") - self.assertEqual(result, 0) - def test_get_multi(self): self.check_setget("gm_a_string", "some random string") self.check_setget("gm_an_integer", 42)