From f3ae57ea9002af6e4f85033c9530f65edb646eaf Mon Sep 17 00:00:00 2001 From: Anand Date: Mon, 23 Apr 2018 18:09:04 +0200 Subject: [PATCH 01/17] Support for PPC --- cpuinfo.py | 1361 +++++++++++++++++++++++++++++++++++++--------------- setup.py | 5 +- 2 files changed, 988 insertions(+), 378 deletions(-) diff --git a/cpuinfo.py b/cpuinfo.py index 463e06ba..c08bbab4 100644 --- a/cpuinfo.py +++ b/cpuinfo.py @@ -1,9 +1,9 @@ #!/usr/bin/env python # -*- coding: UTF-8 -*- -# Copyright (c) 2014-2016, Matthew Brennan Jones -# Py-cpuinfo is a Python module to show the cpuinfo of a processor -# It uses a MIT style license +# Copyright (c) 2014-2018, Matthew Brennan Jones +# Py-cpuinfo gets CPU info with pure Python 2 & 3 +# It uses the MIT License # It is hosted at: https://github.com/workhorsy/py-cpuinfo # # Permission is hereby granted, free of charge, to any person obtaining @@ -25,8 +25,10 @@ # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +CPUINFO_VERSION = (4, 0, 0) import os, sys +import glob import re import time import platform @@ -46,12 +48,45 @@ PY2 = sys.version_info[0] == 2 +# Load hacks for Windows +if platform.system().lower() == 'windows': + # Monkey patch multiprocessing's Popen to fork properly on Windows Pyinstaller + # https://github.com/pyinstaller/pyinstaller/wiki/Recipe-Multiprocessing + try: + import multiprocessing.popen_spawn_win32 as forking + except ImportError as err: + try: + import multiprocessing.popen_fork as forking + except ImportError as err: + import multiprocessing.forking as forking + + class _Popen(forking.Popen): + def __init__(self, *args, **kw): + if hasattr(sys, 'frozen'): + # We have to set original _MEIPASS2 value from sys._MEIPASS + # to get --onefile mode working. + os.putenv('_MEIPASS2', sys._MEIPASS) + try: + super(_Popen, self).__init__(*args, **kw) + finally: + if hasattr(sys, 'frozen'): + # On some platforms (e.g. AIX) 'os.unsetenv()' is not + # available. In those cases we cannot delete the variable + # but only set it to the empty string. The bootloader + # can handle this case. + if hasattr(os, 'unsetenv'): + os.unsetenv('_MEIPASS2') + else: + os.putenv('_MEIPASS2', '') + + forking.Popen = _Popen class DataSource(object): bits = platform.architecture()[0] cpu_count = multiprocessing.cpu_count() is_windows = platform.system().lower() == 'windows' raw_arch_string = platform.machine() + can_cpuid = True @staticmethod def has_proc_cpuinfo(): @@ -62,8 +97,9 @@ def has_dmesg(): return len(program_paths('dmesg')) > 0 @staticmethod - def has_dmesg_boot(): - return os.path.exists('/var/run/dmesg.boot') + def has_var_run_dmesg_boot(): + uname = platform.system().strip().strip('"').strip("'").strip().lower() + return 'linux' in uname and os.path.exists('/var/run/dmesg.boot') @staticmethod def has_cpufreq_info(): @@ -93,6 +129,15 @@ def has_sysinfo(): def has_lscpu(): return len(program_paths('lscpu')) > 0 + @staticmethod + def has_ibm_pa_features(): + return len(program_paths('lsprop')) > 0 + + @staticmethod + def has_wmic(): + returncode, output = run_and_get_stdout(['wmic', 'os', 'get', 'Version']) + return returncode == 0 and len(output) > 0 + @staticmethod def cat_proc_cpuinfo(): return run_and_get_stdout(['cat', '/proc/cpuinfo']) @@ -114,7 +159,7 @@ def dmesg_a(): return run_and_get_stdout(['dmesg', '-a']) @staticmethod - def cat_dmesg_boot(): + def cat_var_run_dmesg_boot(): return run_and_get_stdout(['cat', '/var/run/dmesg.boot']) @staticmethod @@ -137,6 +182,16 @@ def sysinfo_cpu(): def lscpu(): return run_and_get_stdout(['lscpu']) + @staticmethod + def ibm_pa_features(): + ibm_features = glob.glob('/proc/device-tree/cpus/*/ibm,pa-features') + if ibm_features: + return run_and_get_stdout(['lsprop', ibm_features[0]]) + + @staticmethod + def wmic_cpu(): + return run_and_get_stdout(['wmic', 'cpu', 'get', 'Name,CurrentClockSpeed,L2CacheSize,L3CacheSize,Description,Caption,Manufacturer', '/format:list']) + @staticmethod def winreg_processor_brand(): key = winreg.OpenKey(winreg.HKEY_LOCAL_MACHINE, r"Hardware\Description\System\CentralProcessor\0") @@ -181,19 +236,22 @@ def obj_to_b64(thing): return d def b64_to_obj(thing): - a = base64.b64decode(thing) - b = pickle.loads(a) - return b + try: + a = base64.b64decode(thing) + b = pickle.loads(a) + return b + except: + return {} def run_and_get_stdout(command, pipe_command=None): if not pipe_command: - p1 = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p1 = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) output = p1.communicate()[0] if not PY2: output = output.decode(encoding='UTF-8') return p1.returncode, output else: - p1 = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p1 = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) p2 = subprocess.Popen(pipe_command, stdin=p1.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE) p1.stdout.close() output = p2.communicate()[0] @@ -271,46 +329,6 @@ def _get_hz_string_from_brand(processor_brand): return (scale, hz_brand) -def _get_hz_string_from_beagle_bone(): - scale, hz_brand = 1, '0.0' - - if not DataSource.has_cpufreq_info(): - return scale, hz_brand - - returncode, output = DataSource.cpufreq_info() - if returncode != 0: - return (scale, hz_brand) - - hz_brand = output.split('current CPU frequency is')[1].split('.')[0].lower() - - if hz_brand.endswith('mhz'): - scale = 6 - elif hz_brand.endswith('ghz'): - scale = 9 - hz_brand = hz_brand.rstrip('mhz').rstrip('ghz').strip() - hz_brand = to_hz_string(hz_brand) - - return (scale, hz_brand) - -def _get_hz_string_from_lscpu(): - scale, hz_brand = 1, '0.0' - - if not DataSource.has_lscpu(): - return scale, hz_brand - - returncode, output = DataSource.lscpu() - if returncode != 0: - return (scale, hz_brand) - - new_hz = _get_field(False, output, None, None, 'CPU max MHz', 'CPU MHz') - if new_hz == None: - return (scale, hz_brand) - - new_hz = to_hz_string(new_hz) - scale = 6 - - return (scale, new_hz) - def to_friendly_hz(ticks, scale): # Get the raw Hz as a string left, right = to_raw_hz(ticks, scale) @@ -372,6 +390,161 @@ def to_hz_string(ticks): return ticks +def to_friendly_bytes(input): + if not input: + return input + input = "{0}".format(input) + + formats = { + r"^[0-9]+B$" : 'B', + r"^[0-9]+K$" : 'KB', + r"^[0-9]+M$" : 'MB', + r"^[0-9]+G$" : 'GB' + } + + for pattern, friendly_size in formats.items(): + if re.match(pattern, input): + return "{0} {1}".format(input[ : -1].strip(), friendly_size) + + return input + +def _parse_cpu_string(cpu_string): + # Get location of fields at end of string + fields_index = cpu_string.find('(', cpu_string.find('@')) + #print(fields_index) + + # Processor Brand + processor_brand = cpu_string + if fields_index != -1: + processor_brand = cpu_string[0 : fields_index].strip() + #print('processor_brand: ', processor_brand) + + fields = None + if fields_index != -1: + fields = cpu_string[fields_index : ] + #print('fields: ', fields) + + # Hz + scale, hz_brand = _get_hz_string_from_brand(processor_brand) + + # Various fields + vendor_id, stepping, model, family = (None, None, None, None) + if fields: + try: + fields = fields.rsplit('(', 1)[1].split(')')[0].split(',') + fields = [f.strip().lower() for f in fields] + fields = [f.split(':') for f in fields] + fields = [{f[0].strip() : f[1].strip()} for f in fields] + #print('fields: ', fields) + for field in fields: + name = list(field.keys())[0] + value = list(field.values())[0] + #print('name:{0}, value:{1}'.format(name, value)) + if name == 'origin': + vendor_id = value.strip('"') + elif name == 'stepping': + stepping = int(value.lstrip('0x'), 16) + elif name == 'model': + model = int(value.lstrip('0x'), 16) + elif name in ['fam', 'family']: + family = int(value.lstrip('0x'), 16) + except: + #raise + pass + + return (processor_brand, hz_brand, scale, vendor_id, stepping, model, family) + +def _parse_dmesg_output(output): + try: + # Get all the dmesg lines that might contain a CPU string + lines = output.split(' CPU0:')[1:] + \ + output.split(' CPU1:')[1:] + \ + output.split(' CPU:')[1:] + \ + output.split('\nCPU0:')[1:] + \ + output.split('\nCPU1:')[1:] + \ + output.split('\nCPU:')[1:] + lines = [l.split('\n')[0].strip() for l in lines] + + # Convert the lines to CPU strings + cpu_strings = [_parse_cpu_string(l) for l in lines] + + # Find the CPU string that has the most fields + best_string = None + highest_count = 0 + for cpu_string in cpu_strings: + count = sum([n is not None for n in cpu_string]) + if count > highest_count: + highest_count = count + best_string = cpu_string + + # If no CPU string was found, return {} + if not best_string: + return {} + + processor_brand, hz_actual, scale, vendor_id, stepping, model, family = best_string + + # Origin + if ' Origin=' in output: + fields = output[output.find(' Origin=') : ].split('\n')[0] + fields = fields.strip().split() + fields = [n.strip().split('=') for n in fields] + fields = [{n[0].strip().lower() : n[1].strip()} for n in fields] + #print('fields: ', fields) + for field in fields: + name = list(field.keys())[0] + value = list(field.values())[0] + #print('name:{0}, value:{1}'.format(name, value)) + if name == 'origin': + vendor_id = value.strip('"') + elif name == 'stepping': + stepping = int(value.lstrip('0x'), 16) + elif name == 'model': + model = int(value.lstrip('0x'), 16) + elif name in ['fam', 'family']: + family = int(value.lstrip('0x'), 16) + #print('FIELDS: ', (vendor_id, stepping, model, family)) + + # Features + flag_lines = [] + for category in [' Features=', ' Features2=', ' AMD Features=', ' AMD Features2=']: + if category in output: + flag_lines.append(output.split(category)[1].split('\n')[0]) + + flags = [] + for line in flag_lines: + line = line.split('<')[1].split('>')[0].lower() + for flag in line.split(','): + flags.append(flag) + flags.sort() + + # Convert from GHz/MHz string to Hz + scale, hz_advertised = _get_hz_string_from_brand(processor_brand) + + info = { + 'vendor_id' : vendor_id, + 'brand' : processor_brand, + + 'stepping' : stepping, + 'model' : model, + 'family' : family, + 'flags' : flags + } + + if hz_advertised and hz_advertised != '0.0': + info['hz_advertised'] = to_friendly_hz(hz_advertised, scale) + info['hz_actual'] = to_friendly_hz(hz_actual, scale) + + if hz_advertised and hz_advertised != '0.0': + info['hz_advertised_raw'] = to_raw_hz(hz_advertised, scale) + info['hz_actual_raw'] = to_raw_hz(hz_actual, scale) + + return {k: v for k, v in info.items() if v} + except: + #raise + pass + + return {} + def parse_arch(raw_arch_string): arch, bits = None, None raw_arch_string = raw_arch_string.lower() @@ -384,7 +557,7 @@ def parse_arch(raw_arch_string): arch = 'X86_64' bits = 64 # ARM - elif re.match('^armv8-a$', raw_arch_string): + elif re.match('^armv8-a|aarch64$', raw_arch_string): arch = 'ARM_8' bits = 64 elif re.match('^armv7$|^armv7[a-z]$|^armv7-[a-z]$|^armv6[a-z]$', raw_arch_string): @@ -397,7 +570,7 @@ def parse_arch(raw_arch_string): elif re.match('^ppc32$|^prep$|^pmac$|^powermac$', raw_arch_string): arch = 'PPC_32' bits = 32 - elif re.match('^powerpc$|^ppc64$', raw_arch_string): + elif re.match('^powerpc$|^ppc64$|^ppc64le$', raw_arch_string): arch = 'PPC_64' bits = 64 # SPARC @@ -418,14 +591,16 @@ def is_bit_set(reg, bit): class CPUID(object): def __init__(self): - # Figure info if SE Linux is on and in enforcing mode + self.prochandle = None + + # Figure out if SE Linux is on and in enforcing mode self.is_selinux_enforcing = False # Just return if the SE Linux Status Tool is not installed if not DataSource.has_sestatus(): return - # Figure info if we can execute heap and execute memory + # Figure out if we can execute heap and execute memory can_selinux_exec_heap = DataSource.sestatus_allow_execheap() can_selinux_exec_memory = DataSource.sestatus_allow_execmem() self.is_selinux_enforcing = (not can_selinux_exec_heap or not can_selinux_exec_memory) @@ -437,9 +612,13 @@ def _asm_func(self, restype=None, argtypes=(), byte_code=[]): if DataSource.is_windows: # Allocate a memory segment the size of the byte code, and make it executable size = len(byte_code) + # Alloc at least 1 page to ensure we own all pages that we want to change protection on + if size < 0x1000: size = 0x1000 MEM_COMMIT = ctypes.c_ulong(0x1000) - PAGE_EXECUTE_READWRITE = ctypes.c_ulong(0x40) - address = ctypes.windll.kernel32.VirtualAlloc(ctypes.c_int(0), ctypes.c_size_t(size), MEM_COMMIT, PAGE_EXECUTE_READWRITE) + PAGE_READWRITE = ctypes.c_ulong(0x4) + pfnVirtualAlloc = ctypes.windll.kernel32.VirtualAlloc + pfnVirtualAlloc.restype = ctypes.c_void_p + address = pfnVirtualAlloc(None, ctypes.c_size_t(size), MEM_COMMIT, PAGE_READWRITE) if not address: raise Exception("Failed to VirtualAlloc") @@ -447,27 +626,48 @@ def _asm_func(self, restype=None, argtypes=(), byte_code=[]): memmove = ctypes.CFUNCTYPE(ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_size_t)(ctypes._memmove_addr) if memmove(address, byte_code, size) < 0: raise Exception("Failed to memmove") + + # Enable execute permissions + PAGE_EXECUTE = ctypes.c_ulong(0x10) + old_protect = ctypes.c_ulong(0) + pfnVirtualProtect = ctypes.windll.kernel32.VirtualProtect + res = pfnVirtualProtect(ctypes.c_void_p(address), ctypes.c_size_t(size), PAGE_EXECUTE, ctypes.byref(old_protect)) + if not res: + raise Exception("Failed VirtualProtect") + + # Flush Instruction Cache + # First, get process Handle + if not self.prochandle: + pfnGetCurrentProcess = ctypes.windll.kernel32.GetCurrentProcess + pfnGetCurrentProcess.restype = ctypes.c_void_p + self.prochandle = ctypes.c_void_p(pfnGetCurrentProcess()) + # Actually flush cache + res = ctypes.windll.kernel32.FlushInstructionCache(self.prochandle, ctypes.c_void_p(address), ctypes.c_size_t(size)) + if not res: + raise Exception("Failed FlushInstructionCache") else: # Allocate a memory segment the size of the byte code size = len(byte_code) - address = ctypes.pythonapi.valloc(size) + pfnvalloc = ctypes.pythonapi.valloc + pfnvalloc.restype = ctypes.c_void_p + address = pfnvalloc(ctypes.c_size_t(size)) if not address: raise Exception("Failed to valloc") # Mark the memory segment as writeable only if not self.is_selinux_enforcing: WRITE = 0x2 - if ctypes.pythonapi.mprotect(address, size, WRITE) < 0: + if ctypes.pythonapi.mprotect(ctypes.c_void_p(address), size, WRITE) < 0: raise Exception("Failed to mprotect") # Copy the byte code into the memory segment - if ctypes.pythonapi.memmove(address, byte_code, size) < 0: + if ctypes.pythonapi.memmove(ctypes.c_void_p(address), byte_code, ctypes.c_size_t(size)) < 0: raise Exception("Failed to memmove") # Mark the memory segment as writeable and executable only if not self.is_selinux_enforcing: WRITE_EXECUTE = 0x2 | 0x4 - if ctypes.pythonapi.mprotect(address, size, WRITE_EXECUTE) < 0: + if ctypes.pythonapi.mprotect(ctypes.c_void_p(address), size, WRITE_EXECUTE) < 0: raise Exception("Failed to mprotect") # Cast the memory segment into a function @@ -477,55 +677,45 @@ def _asm_func(self, restype=None, argtypes=(), byte_code=[]): def _run_asm(self, *byte_code): # Convert the byte code into a function that returns an int - restype = None - if DataSource.bits == '64bit': - restype = ctypes.c_uint64 - else: - restype = ctypes.c_uint32 + restype = ctypes.c_uint32 argtypes = () func, address = self._asm_func(restype, argtypes, byte_code) # Call the byte code like a function retval = func() + byte_code = bytes.join(b'', byte_code) size = ctypes.c_size_t(len(byte_code)) # Free the function memory segment if DataSource.is_windows: MEM_RELEASE = ctypes.c_ulong(0x8000) - ctypes.windll.kernel32.VirtualFree(address, size, MEM_RELEASE) + ctypes.windll.kernel32.VirtualFree(ctypes.c_void_p(address), ctypes.c_size_t(0), MEM_RELEASE) else: # Remove the executable tag on the memory READ_WRITE = 0x1 | 0x2 - if ctypes.pythonapi.mprotect(address, size, READ_WRITE) < 0: + if ctypes.pythonapi.mprotect(ctypes.c_void_p(address), size, READ_WRITE) < 0: raise Exception("Failed to mprotect") - ctypes.pythonapi.free(address) + ctypes.pythonapi.free(ctypes.c_void_p(address)) return retval # FIXME: We should not have to use different instructions to # set eax to 0 or 1, on 32bit and 64bit machines. def _zero_eax(self): - if DataSource.bits == '64bit': - return ( - b"\x66\xB8\x00\x00" # mov eax,0x0" - ) - else: - return ( - b"\x31\xC0" # xor ax,ax - ) + return ( + b"\x31\xC0" # xor eax,eax + ) + def _zero_ecx(self): + return ( + b"\x31\xC9" # xor ecx,ecx + ) def _one_eax(self): - if DataSource.bits == '64bit': - return ( - b"\x66\xB8\x01\x00" # mov eax,0x1" - ) - else: - return ( - b"\x31\xC0" # xor ax,ax - b"\x40" # inc ax - ) + return ( + b"\xB8\x01\x00\x00\x00" # mov eax,0x1" + ) # http://en.wikipedia.org/wiki/CPUID#EAX.3D0:_Get_vendor_ID def get_vendor_id(self): @@ -588,7 +778,7 @@ def get_info(self): 'extended_family' : extended_family } - # https://en.wikipedia.org/wiki/CPUID#EAX.3D80000000h:_Get_Highest_Extended_Function_Supported + # http://en.wikipedia.org/wiki/CPUID#EAX.3D80000000h:_Get_Highest_Extended_Function_Supported def get_max_extension_support(self): # Check for extension support max_extension_support = self._run_asm( @@ -689,18 +879,102 @@ def get_flags(self, max_extension_support): # Get a list of only the flags that are true flags = [k for k, v in flags.items() if v] - # Get the Extended CPU flags - extended_flags = {} + # http://en.wikipedia.org/wiki/CPUID#EAX.3D7.2C_ECX.3D0:_Extended_Features + if max_extension_support >= 7: + # EBX + ebx = self._run_asm( + self._zero_ecx(), + b"\xB8\x07\x00\x00\x00" # mov eax,7 + b"\x0f\xa2" # cpuid + b"\x89\xD8" # mov ax,bx + b"\xC3" # ret + ) - # https://en.wikipedia.org/wiki/CPUID#EAX.3D7.2C_ECX.3D0:_Extended_Features - if max_extension_support == 7: - pass - # FIXME: Are we missing all these flags too? - # avx2 et cetera ... + # ECX + ecx = self._run_asm( + self._zero_ecx(), + b"\xB8\x07\x00\x00\x00" # mov eax,7 + b"\x0f\xa2" # cpuid + b"\x89\xC8" # mov ax,cx + b"\xC3" # ret + ) + + # Get the extended CPU flags + extended_flags = { + #'fsgsbase' : is_bit_set(ebx, 0), + #'IA32_TSC_ADJUST' : is_bit_set(ebx, 1), + 'sgx' : is_bit_set(ebx, 2), + 'bmi1' : is_bit_set(ebx, 3), + 'hle' : is_bit_set(ebx, 4), + 'avx2' : is_bit_set(ebx, 5), + #'reserved' : is_bit_set(ebx, 6), + 'smep' : is_bit_set(ebx, 7), + 'bmi2' : is_bit_set(ebx, 8), + 'erms' : is_bit_set(ebx, 9), + 'invpcid' : is_bit_set(ebx, 10), + 'rtm' : is_bit_set(ebx, 11), + 'pqm' : is_bit_set(ebx, 12), + #'FPU CS and FPU DS deprecated' : is_bit_set(ebx, 13), + 'mpx' : is_bit_set(ebx, 14), + 'pqe' : is_bit_set(ebx, 15), + 'avx512f' : is_bit_set(ebx, 16), + 'avx512dq' : is_bit_set(ebx, 17), + 'rdseed' : is_bit_set(ebx, 18), + 'adx' : is_bit_set(ebx, 19), + 'smap' : is_bit_set(ebx, 20), + 'avx512ifma' : is_bit_set(ebx, 21), + 'pcommit' : is_bit_set(ebx, 22), + 'clflushopt' : is_bit_set(ebx, 23), + 'clwb' : is_bit_set(ebx, 24), + 'intel_pt' : is_bit_set(ebx, 25), + 'avx512pf' : is_bit_set(ebx, 26), + 'avx512er' : is_bit_set(ebx, 27), + 'avx512cd' : is_bit_set(ebx, 28), + 'sha' : is_bit_set(ebx, 29), + 'avx512bw' : is_bit_set(ebx, 30), + 'avx512vl' : is_bit_set(ebx, 31), + + 'prefetchwt1' : is_bit_set(ecx, 0), + 'avx512vbmi' : is_bit_set(ecx, 1), + 'umip' : is_bit_set(ecx, 2), + 'pku' : is_bit_set(ecx, 3), + 'ospke' : is_bit_set(ecx, 4), + #'reserved' : is_bit_set(ecx, 5), + 'avx512vbmi2' : is_bit_set(ecx, 6), + #'reserved' : is_bit_set(ecx, 7), + 'gfni' : is_bit_set(ecx, 8), + 'vaes' : is_bit_set(ecx, 9), + 'vpclmulqdq' : is_bit_set(ecx, 10), + 'avx512vnni' : is_bit_set(ecx, 11), + 'avx512bitalg' : is_bit_set(ecx, 12), + #'reserved' : is_bit_set(ecx, 13), + 'avx512vpopcntdq' : is_bit_set(ecx, 14), + #'reserved' : is_bit_set(ecx, 15), + #'reserved' : is_bit_set(ecx, 16), + #'mpx0' : is_bit_set(ecx, 17), + #'mpx1' : is_bit_set(ecx, 18), + #'mpx2' : is_bit_set(ecx, 19), + #'mpx3' : is_bit_set(ecx, 20), + #'mpx4' : is_bit_set(ecx, 21), + 'rdpid' : is_bit_set(ecx, 22), + #'reserved' : is_bit_set(ecx, 23), + #'reserved' : is_bit_set(ecx, 24), + #'reserved' : is_bit_set(ecx, 25), + #'reserved' : is_bit_set(ecx, 26), + #'reserved' : is_bit_set(ecx, 27), + #'reserved' : is_bit_set(ecx, 28), + #'reserved' : is_bit_set(ecx, 29), + 'sgx_lc' : is_bit_set(ecx, 30), + #'reserved' : is_bit_set(ecx, 31) + } + + # Get a list of only the flags that are true + extended_flags = [k for k, v in extended_flags.items() if v] + flags += extended_flags - # https://en.wikipedia.org/wiki/CPUID#EAX.3D80000001h:_Extended_Processor_Info_and_Feature_Bits + # http://en.wikipedia.org/wiki/CPUID#EAX.3D80000001h:_Extended_Processor_Info_and_Feature_Bits if max_extension_support >= 0x80000001: - # EBX # FIXME: This may need to be EDX instead + # EBX ebx = self._run_asm( b"\xB8\x01\x00\x00\x80" # mov ax,0x80000001 b"\x0f\xa2" # cpuid @@ -785,14 +1059,14 @@ def get_flags(self, max_extension_support): #'reserved' : is_bit_set(ecx, 31) } - # Get a list of only the flags that are true - extended_flags = [k for k, v in extended_flags.items() if v] - flags += extended_flags + # Get a list of only the flags that are true + extended_flags = [k for k, v in extended_flags.items() if v] + flags += extended_flags flags.sort() return flags - # https://en.wikipedia.org/wiki/CPUID#EAX.3D80000002h.2C80000003h.2C80000004h:_Processor_Brand_String + # http://en.wikipedia.org/wiki/CPUID#EAX.3D80000002h.2C80000003h.2C80000004h:_Processor_Brand_String def get_processor_brand(self, max_extension_support): processor_brand = "" @@ -846,7 +1120,7 @@ def get_processor_brand(self, max_extension_support): return processor_brand - # https://en.wikipedia.org/wiki/CPUID#EAX.3D80000006h:_Extended_L2_Cache_Features + # http://en.wikipedia.org/wiki/CPUID#EAX.3D80000006h:_Extended_L2_Cache_Features def get_cache(self, max_extension_support): cache_info = {} @@ -930,31 +1204,30 @@ def get_raw_hz(self): return ticks -def get_cpu_info_from_cpuid(): +def _actual_get_cpu_info_from_cpuid(queue): ''' - Returns the CPU info gathered by querying the X86 cpuid register in a new process. - Returns None of non X86 cpus. - Returns None if SELinux is in enforcing mode. + Warning! This function has the potential to crash the Python runtime. + Do not call it directly. Use the _get_cpu_info_from_cpuid function instead. + It will safely call this function in another process. ''' - returncode, output = run_and_get_stdout([sys.executable, "-c", "import cpuinfo; print(cpuinfo.actual_get_cpu_info_from_cpuid())"]) - if returncode != 0: - return None - info = b64_to_obj(output) - return info + # Pipe all output to nothing + sys.stdout = open(os.devnull, 'w') + sys.stderr = open(os.devnull, 'w') -def actual_get_cpu_info_from_cpuid(): # Get the CPU arch and bits arch, bits = parse_arch(DataSource.raw_arch_string) # Return none if this is not an X86 CPU if not arch in ['X86_32', 'X86_64']: - return None + queue.put(obj_to_b64({})) + return # Return none if SE Linux is in enforcing mode cpuid = CPUID() if cpuid.is_selinux_enforcing: - return None + queue.put(obj_to_b64({})) + return # Get the cpu info from the CPUID register max_extension_support = cpuid.get_max_extension_support() @@ -969,23 +1242,17 @@ def actual_get_cpu_info_from_cpuid(): # Get the Hz and scale scale, hz_advertised = _get_hz_string_from_brand(processor_brand) - info = { 'vendor_id' : cpuid.get_vendor_id(), 'hardware' : '', 'brand' : processor_brand, 'hz_advertised' : to_friendly_hz(hz_advertised, scale), - 'hz_actual' : to_friendly_hz(hz_actual, 6), + 'hz_actual' : to_friendly_hz(hz_actual, 0), 'hz_advertised_raw' : to_raw_hz(hz_advertised, scale), - 'hz_actual_raw' : to_raw_hz(hz_actual, 6), - - 'arch' : arch, - 'bits' : bits, - 'count' : DataSource.cpu_count, - 'raw_arch_string' : DataSource.raw_arch_string, + 'hz_actual_raw' : to_raw_hz(hz_actual, 0), - 'l2_cache_size' : cache_info['size_kb'], + 'l2_cache_size' : to_friendly_bytes(cache_info['size_kb']), 'l2_cache_line_size' : cache_info['line_size_b'], 'l2_cache_associativity' : hex(cache_info['associativity']), @@ -997,21 +1264,66 @@ def actual_get_cpu_info_from_cpuid(): 'extended_family' : info['extended_family'], 'flags' : cpuid.get_flags(max_extension_support) } - return obj_to_b64(info) -def get_cpu_info_from_proc_cpuinfo(): + info = {k: v for k, v in info.items() if v} + queue.put(obj_to_b64(info)) + +def _get_cpu_info_from_cpuid(): + ''' + Returns the CPU info gathered by querying the X86 cpuid register in a new process. + Returns {} on non X86 cpus. + Returns {} if SELinux is in enforcing mode. + ''' + from multiprocessing import Process, Queue + + # Return {} if can't cpuid + if not DataSource.can_cpuid: + return {} + + # Get the CPU arch and bits + arch, bits = parse_arch(DataSource.raw_arch_string) + + # Return {} if this is not an X86 CPU + if not arch in ['X86_32', 'X86_64']: + return {} + + try: + # Start running the function in a subprocess + queue = Queue() + p = Process(target=_actual_get_cpu_info_from_cpuid, args=(queue,)) + p.start() + + # Wait for the process to end, while it is still alive + while p.is_alive(): + p.join(0) + + # Return {} if it failed + if p.exitcode != 0: + return {} + + # Return the result, only if there is something to read + if not queue.empty(): + output = queue.get() + return b64_to_obj(output) + except: + pass + + # Return {} if everything failed + return {} + +def _get_cpu_info_from_proc_cpuinfo(): ''' - Returns the CPU info gathered from /proc/cpuinfo. Will return None if - /proc/cpuinfo is not found. + Returns the CPU info gathered from /proc/cpuinfo. + Returns {} if /proc/cpuinfo is not found. ''' try: - # Just return None if there is no cpuinfo + # Just return {} if there is no cpuinfo if not DataSource.has_proc_cpuinfo(): - return None + return {} returncode, output = DataSource.cat_proc_cpuinfo() if returncode != 0: - return None + return {} # Various fields vendor_id = _get_field(False, output, None, '', 'vendor_id', 'vendor id', 'vendor') @@ -1022,8 +1334,10 @@ def get_cpu_info_from_proc_cpuinfo(): family = _get_field(False, output, int, 0, 'cpu family') hardware = _get_field(False, output, None, '', 'Hardware') # Flags - flags = _get_field(False, output, None, None, 'flags', 'Features').split() - flags.sort() + flags = _get_field(False, output, None, None, 'flags', 'Features') + if flags: + flags = flags.split() + flags.sort() # Convert from MHz string to Hz hz_actual = _get_field(False, output, None, '', 'cpu MHz', 'cpu speed', 'clock') @@ -1031,188 +1345,329 @@ def get_cpu_info_from_proc_cpuinfo(): hz_actual = to_hz_string(hz_actual) # Convert from GHz/MHz string to Hz - scale, hz_advertised = _get_hz_string_from_brand(processor_brand) + scale, hz_advertised = (0, None) + try: + scale, hz_advertised = _get_hz_string_from_brand(processor_brand) + except Exception: + pass - # Try getting the Hz for a BeagleBone - if hz_advertised == '0.0': - scale, hz_advertised = _get_hz_string_from_beagle_bone() - hz_actual = hz_advertised + info = { + 'hardware' : hardware, + 'brand' : processor_brand, + + 'l3_cache_size' : to_friendly_bytes(cache_size), + 'flags' : flags, + 'vendor_id' : vendor_id, + 'stepping' : stepping, + 'model' : model, + 'family' : family, + } - # Try getting the Hz for a lscpu - if hz_advertised == '0.0': - scale, hz_advertised = _get_hz_string_from_lscpu() + # Make the Hz the same for actual and advertised if missing any + if not hz_advertised or hz_advertised == '0.0': + hz_advertised = hz_actual + scale = 6 + elif not hz_actual or hz_actual == '0.0': hz_actual = hz_advertised - # Get the CPU arch and bits - arch, bits = parse_arch(DataSource.raw_arch_string) + # Add the Hz if there is one + if to_raw_hz(hz_advertised, scale) > (0, 0): + info['hz_advertised'] = to_friendly_hz(hz_advertised, scale) + info['hz_advertised_raw'] = to_raw_hz(hz_advertised, scale) + if to_raw_hz(hz_actual, scale) > (0, 0): + info['hz_actual'] = to_friendly_hz(hz_actual, 6) + info['hz_actual_raw'] = to_raw_hz(hz_actual, 6) - return { - 'vendor_id' : vendor_id, - 'hardware' : hardware, - 'brand' : processor_brand, + info = {k: v for k, v in info.items() if v} + return info + except: + #raise # NOTE: To have this throw on error, uncomment this line + return {} - 'hz_advertised' : to_friendly_hz(hz_advertised, scale), - 'hz_actual' : to_friendly_hz(hz_actual, 6), - 'hz_advertised_raw' : to_raw_hz(hz_advertised, scale), - 'hz_actual_raw' : to_raw_hz(hz_actual, 6), +def _get_cpu_info_from_cpufreq_info(): + ''' + Returns the CPU info gathered from cpufreq-info. + Returns {} if cpufreq-info is not found. + ''' + try: + scale, hz_brand = 1, '0.0' - 'arch' : arch, - 'bits' : bits, - 'count' : DataSource.cpu_count, - 'raw_arch_string' : DataSource.raw_arch_string, + if not DataSource.has_cpufreq_info(): + return {} - 'l2_cache_size' : cache_size, - 'l2_cache_line_size' : 0, - 'l2_cache_associativity' : 0, + returncode, output = DataSource.cpufreq_info() + if returncode != 0: + return {} - 'stepping' : stepping, - 'model' : model, - 'family' : family, - 'processor_type' : 0, - 'extended_model' : 0, - 'extended_family' : 0, - 'flags' : flags + hz_brand = output.split('current CPU frequency is')[1].split('\n')[0] + i = hz_brand.find('Hz') + assert(i != -1) + hz_brand = hz_brand[0 : i+2].strip().lower() + + if hz_brand.endswith('mhz'): + scale = 6 + elif hz_brand.endswith('ghz'): + scale = 9 + hz_brand = hz_brand.rstrip('mhz').rstrip('ghz').strip() + hz_brand = to_hz_string(hz_brand) + + info = { + 'hz_advertised' : to_friendly_hz(hz_brand, scale), + 'hz_actual' : to_friendly_hz(hz_brand, scale), + 'hz_advertised_raw' : to_raw_hz(hz_brand, scale), + 'hz_actual_raw' : to_raw_hz(hz_brand, scale), } + + info = {k: v for k, v in info.items() if v} + return info except: #raise # NOTE: To have this throw on error, uncomment this line - return None + return {} -def parse_dmesg_output(output): +def _get_cpu_info_from_lscpu(): ''' - Parse dmesg output. Return None if there are any errors. + Returns the CPU info gathered from lscpu. + Returns {} if lscpu is not found. ''' try: - # Processor Brand - long_brand = output.split('CPU: ')[1].split('\n')[0] - processor_brand = long_brand.rsplit('(', 1)[0] - processor_brand = processor_brand.strip() - - # Hz - scale = 0 - hz_actual = long_brand.rsplit('(', 1)[1].split(' ')[0].lower() - if hz_actual.endswith('mhz'): + if not DataSource.has_lscpu(): + return {} + + returncode, output = DataSource.lscpu() + if returncode != 0: + return {} + + info = {} + + new_hz = _get_field(False, output, None, None, 'CPU max MHz', 'CPU MHz') + if new_hz: + new_hz = to_hz_string(new_hz) scale = 6 - elif hz_actual.endswith('ghz'): - scale = 9 - hz_actual = hz_actual.split('-')[0] - hz_actual = to_hz_string(hz_actual) + info['hz_advertised'] = to_friendly_hz(new_hz, scale) + info['hz_actual'] = to_friendly_hz(new_hz, scale) + info['hz_advertised_raw'] = to_raw_hz(new_hz, scale) + info['hz_actual_raw'] = to_raw_hz(new_hz, scale) - # Various fields - fields = output.split('CPU: ')[1].split('\n')[1].split('\n')[0].strip().split(' ') - vendor_id = None - stepping = None - model = None - family = None - for field in fields: - name, value = field.split('=') - name = name.strip().lower() - value = value.strip() - if name == 'origin': - vendor_id = value.strip('"') - elif name == 'stepping': - stepping = int(value) - elif name == 'model': - model = int(value, 16) - elif name == 'family': - family = int(value, 16) + vendor_id = _get_field(False, output, None, None, 'Vendor ID') + if vendor_id: + info['vendor_id'] = vendor_id - # Flags - flag_lines = [] - for category in [' Features=', ' Features2=', ' AMD Features=', ' AMD Features2=']: - if category in output: - flag_lines.append(output.split(category)[1].split('\n')[0]) + brand = _get_field(False, output, None, None, 'Model name') + if brand: + info['brand'] = brand - flags = [] - for line in flag_lines: - line = line.split('<')[1].split('>')[0].lower() - for flag in line.split(','): - flags.append(flag) - flags.sort() + family = _get_field(False, output, None, None, 'CPU family') + if family and family.isdigit(): + info['family'] = int(family) - # Convert from GHz/MHz string to Hz - scale, hz_advertised = _get_hz_string_from_brand(processor_brand) + stepping = _get_field(False, output, None, None, 'Stepping') + if stepping and stepping.isdigit(): + info['stepping'] = int(stepping) - # Get the CPU arch and bits - arch, bits = parse_arch(DataSource.raw_arch_string) + model = _get_field(False, output, None, None, 'Model') + if model and model.isdigit(): + info['model'] = int(model) - return { - 'vendor_id' : vendor_id, - 'hardware' : '', - 'brand' : processor_brand, + l1_data_cache_size = _get_field(False, output, None, None, 'L1d cache') + if l1_data_cache_size: + info['l1_data_cache_size'] = to_friendly_bytes(l1_data_cache_size) - 'hz_advertised' : to_friendly_hz(hz_advertised, scale), - 'hz_actual' : to_friendly_hz(hz_actual, 6), - 'hz_advertised_raw' : to_raw_hz(hz_advertised, scale), - 'hz_actual_raw' : to_raw_hz(hz_actual, 6), + l1_instruction_cache_size = _get_field(False, output, None, None, 'L1i cache') + if l1_instruction_cache_size: + info['l1_instruction_cache_size'] = to_friendly_bytes(l1_instruction_cache_size) - 'arch' : arch, - 'bits' : bits, - 'count' : DataSource.cpu_count, - 'raw_arch_string' : DataSource.raw_arch_string, + l2_cache_size = _get_field(False, output, None, None, 'L2 cache') + if l2_cache_size: + info['l2_cache_size'] = to_friendly_bytes(l2_cache_size) - 'l2_cache_size' : 0, - 'l2_cache_line_size' : 0, - 'l2_cache_associativity' : 0, + l3_cache_size = _get_field(False, output, None, None, 'L3 cache') + if l3_cache_size: + info['l3_cache_size'] = to_friendly_bytes(l3_cache_size) - 'stepping' : stepping, - 'model' : model, - 'family' : family, - 'processor_type' : 0, - 'extended_model' : 0, - 'extended_family' : 0, - 'flags' : flags - } + # Flags + flags = _get_field(False, output, None, None, 'flags', 'Features') + if flags: + flags = flags.split() + flags.sort() + info['flags'] = flags + + info = {k: v for k, v in info.items() if v} + return info except: - return None - + #raise # NOTE: To have this throw on error, uncomment this line + return {} -def get_cpu_info_from_dmesg(): +def _get_cpu_info_from_dmesg(): ''' - Returns the CPU info gathered from dmesg. Will return None if - dmesg is not found or does not have the desired info. + Returns the CPU info gathered from dmesg. + Returns {} if dmesg is not found or does not have the desired info. ''' - # Just return None if there is no dmesg + # Just return {} if there is no dmesg if not DataSource.has_dmesg(): - return None + return {} - # If dmesg fails return None + # If dmesg fails return {} returncode, output = DataSource.dmesg_a() if output == None or returncode != 0: - return None + return {} + + return _parse_dmesg_output(output) + + +# https://openpowerfoundation.org/wp-content/uploads/2016/05/LoPAPR_DRAFT_v11_24March2016_cmt1.pdf +# page 767 +def _get_cpu_info_from_ibm_pa_features(): + ''' + Returns the CPU info gathered from lsprop /proc/device-tree/cpus/*/ibm,pa-features + Returns {} if lsprop is not found or ibm,pa-features does not have the desired info. + ''' + try: + # Just return {} if there is no lsprop + if not DataSource.has_ibm_pa_features(): + return {} + + # If ibm,pa-features fails return {} + returncode, output = DataSource.ibm_pa_features() + if output == None or returncode != 0: + return {} + + # Filter out invalid characters from output + value = output.split("ibm,pa-features")[1].lower() + value = [s for s in value if s in list('0123456789abcfed')] + value = ''.join(value) - return parse_dmesg_output(output) + # Get data converted to Uint32 chunks + left = int(value[0 : 8], 16) + right = int(value[8 : 16], 16) + # Get the CPU flags + flags = { + # Byte 0 + 'mmu' : is_bit_set(left, 0), + 'fpu' : is_bit_set(left, 1), + 'slb' : is_bit_set(left, 2), + 'run' : is_bit_set(left, 3), + #'reserved' : is_bit_set(left, 4), + 'dabr' : is_bit_set(left, 5), + 'ne' : is_bit_set(left, 6), + 'wtr' : is_bit_set(left, 7), + + # Byte 1 + 'mcr' : is_bit_set(left, 8), + 'dsisr' : is_bit_set(left, 9), + 'lp' : is_bit_set(left, 10), + 'ri' : is_bit_set(left, 11), + 'dabrx' : is_bit_set(left, 12), + 'sprg3' : is_bit_set(left, 13), + 'rislb' : is_bit_set(left, 14), + 'pp' : is_bit_set(left, 15), + + # Byte 2 + 'vpm' : is_bit_set(left, 16), + 'dss_2.05' : is_bit_set(left, 17), + #'reserved' : is_bit_set(left, 18), + 'dar' : is_bit_set(left, 19), + #'reserved' : is_bit_set(left, 20), + 'ppr' : is_bit_set(left, 21), + 'dss_2.02' : is_bit_set(left, 22), + 'dss_2.06' : is_bit_set(left, 23), + + # Byte 3 + 'lsd_in_dscr' : is_bit_set(left, 24), + 'ugr_in_dscr' : is_bit_set(left, 25), + #'reserved' : is_bit_set(left, 26), + #'reserved' : is_bit_set(left, 27), + #'reserved' : is_bit_set(left, 28), + #'reserved' : is_bit_set(left, 29), + #'reserved' : is_bit_set(left, 30), + #'reserved' : is_bit_set(left, 31), + + # Byte 4 + 'sso_2.06' : is_bit_set(right, 0), + #'reserved' : is_bit_set(right, 1), + #'reserved' : is_bit_set(right, 2), + #'reserved' : is_bit_set(right, 3), + #'reserved' : is_bit_set(right, 4), + #'reserved' : is_bit_set(right, 5), + #'reserved' : is_bit_set(right, 6), + #'reserved' : is_bit_set(right, 7), + + # Byte 5 + 'le' : is_bit_set(right, 8), + 'cfar' : is_bit_set(right, 9), + 'eb' : is_bit_set(right, 10), + 'lsq_2.07' : is_bit_set(right, 11), + #'reserved' : is_bit_set(right, 12), + #'reserved' : is_bit_set(right, 13), + #'reserved' : is_bit_set(right, 14), + #'reserved' : is_bit_set(right, 15), + + # Byte 6 + 'dss_2.07' : is_bit_set(right, 16), + #'reserved' : is_bit_set(right, 17), + #'reserved' : is_bit_set(right, 18), + #'reserved' : is_bit_set(right, 19), + #'reserved' : is_bit_set(right, 20), + #'reserved' : is_bit_set(right, 21), + #'reserved' : is_bit_set(right, 22), + #'reserved' : is_bit_set(right, 23), + + # Byte 7 + #'reserved' : is_bit_set(right, 24), + #'reserved' : is_bit_set(right, 25), + #'reserved' : is_bit_set(right, 26), + #'reserved' : is_bit_set(right, 27), + #'reserved' : is_bit_set(right, 28), + #'reserved' : is_bit_set(right, 29), + #'reserved' : is_bit_set(right, 30), + #'reserved' : is_bit_set(right, 31), + } + + # Get a list of only the flags that are true + flags = [k for k, v in flags.items() if v] + flags.sort() -def get_cpu_info_from_dmesg_boot(): - ''' - Returns CPU info gathered from dmesg during boot. Will return None if - dmesg is not found or does not have the desired info. - ''' + info = { + 'flags' : flags + } + info = {k: v for k, v in info.items() if v} - if not DataSource.has_dmesg_boot(): - return None + return info + except: + return {} - returncode, output = DataSource.cat_dmesg_boot() - if output == None or returncode != 0: - return None - return parse_dmesg_output(output) +def _get_cpu_info_from_cat_var_run_dmesg_boot(): + ''' + Returns the CPU info gathered from /var/run/dmesg.boot. + Returns {} if dmesg is not found or does not have the desired info. + ''' + # Just return {} if there is no /var/run/dmesg.boot + if not DataSource.has_var_run_dmesg_boot(): + return {} + # If dmesg.boot fails return {} + returncode, output = DataSource.cat_var_run_dmesg_boot() + if output == None or returncode != 0: + return {} -def get_cpu_info_from_sysctl(): + return _parse_dmesg_output(output) + + +def _get_cpu_info_from_sysctl(): ''' - Returns the CPU info gathered from sysctl. Will return None if - sysctl is not found. + Returns the CPU info gathered from sysctl. + Returns {} if sysctl is not found. ''' try: - # Just return None if there is no sysctl + # Just return {} if there is no sysctl if not DataSource.has_sysctl(): - return None + return {} - # If sysctl fails return None + # If sysctl fails return {} returncode, output = DataSource.sysctl_machdep_cpu_hw_cpufrequency() if output == None or returncode != 0: - return None + return {} # Various fields vendor_id = _get_field(False, output, None, None, 'machdep.cpu.vendor') @@ -1223,7 +1678,9 @@ def get_cpu_info_from_sysctl(): family = _get_field(False, output, int, 0, 'machdep.cpu.family') # Flags - flags = _get_field(False, output, None, None, 'machdep.cpu.features').lower().split() + flags = _get_field(False, output, None, '', 'machdep.cpu.features').lower().split() + flags.extend(_get_field(False, output, None, '', 'machdep.cpu.leaf7_features').lower().split()) + flags.extend(_get_field(False, output, None, '', 'machdep.cpu.extfeatures').lower().split()) flags.sort() # Convert from GHz/MHz string to Hz @@ -1231,12 +1688,8 @@ def get_cpu_info_from_sysctl(): hz_actual = _get_field(False, output, None, None, 'hw.cpufrequency') hz_actual = to_hz_string(hz_actual) - # Get the CPU arch and bits - arch, bits = parse_arch(DataSource.raw_arch_string) - - return { + info = { 'vendor_id' : vendor_id, - 'hardware' : '', 'brand' : processor_brand, 'hz_advertised' : to_friendly_hz(hz_advertised, scale), @@ -1244,40 +1697,43 @@ def get_cpu_info_from_sysctl(): 'hz_advertised_raw' : to_raw_hz(hz_advertised, scale), 'hz_actual_raw' : to_raw_hz(hz_actual, 0), - 'arch' : arch, - 'bits' : bits, - 'count' : DataSource.cpu_count, - 'raw_arch_string' : DataSource.raw_arch_string, - - 'l2_cache_size' : cache_size, - 'l2_cache_line_size' : 0, - 'l2_cache_associativity' : 0, + 'l2_cache_size' : to_friendly_bytes(cache_size), 'stepping' : stepping, 'model' : model, 'family' : family, - 'processor_type' : 0, - 'extended_model' : 0, - 'extended_family' : 0, 'flags' : flags } + + info = {k: v for k, v in info.items() if v} + return info except: - return None + return {} -def get_cpu_info_from_sysinfo(): + +def _get_cpu_info_from_sysinfo(): + ''' + Returns the CPU info gathered from sysinfo. + Returns {} if sysinfo is not found. + ''' + info = _get_cpu_info_from_sysinfo_v1() + info.update(_get_cpu_info_from_sysinfo_v2()) + return info + +def _get_cpu_info_from_sysinfo_v1(): ''' - Returns the CPU info gathered from sysinfo. Will return None if - sysinfo is not found. + Returns the CPU info gathered from sysinfo. + Returns {} if sysinfo is not found. ''' try: - # Just return None if there is no sysinfo + # Just return {} if there is no sysinfo if not DataSource.has_sysinfo(): - return None + return {} - # If sysinfo fails return None + # If sysinfo fails return {} returncode, output = DataSource.sysinfo_cpu() if output == None or returncode != 0: - return None + return {} # Various fields vendor_id = '' #_get_field(False, output, None, None, 'CPU #0: ') @@ -1299,12 +1755,8 @@ def get_cpu_info_from_sysinfo(): scale, hz_advertised = _get_hz_string_from_brand(processor_brand) hz_actual = hz_advertised - # Get the CPU arch and bits - arch, bits = parse_arch(DataSource.raw_arch_string) - - return { + info = { 'vendor_id' : vendor_id, - 'hardware' : '', 'brand' : processor_brand, 'hz_advertised' : to_friendly_hz(hz_advertised, scale), @@ -1312,36 +1764,173 @@ def get_cpu_info_from_sysinfo(): 'hz_advertised_raw' : to_raw_hz(hz_advertised, scale), 'hz_actual_raw' : to_raw_hz(hz_actual, scale), - 'arch' : arch, - 'bits' : bits, - 'count' : DataSource.cpu_count, - 'raw_arch_string' : DataSource.raw_arch_string, + 'l2_cache_size' : to_friendly_bytes(cache_size), - 'l2_cache_size' : cache_size, - 'l2_cache_line_size' : 0, - 'l2_cache_associativity' : 0, + 'stepping' : stepping, + 'model' : model, + 'family' : family, + 'flags' : flags + } + + info = {k: v for k, v in info.items() if v} + return info + except: + return {} + +def _get_cpu_info_from_sysinfo_v2(): + ''' + Returns the CPU info gathered from sysinfo. + Returns {} if sysinfo is not found. + ''' + try: + # Just return {} if there is no sysinfo + if not DataSource.has_sysinfo(): + return {} + + # If sysinfo fails return {} + returncode, output = DataSource.sysinfo_cpu() + if output == None or returncode != 0: + return {} + + # Various fields + vendor_id = '' #_get_field(False, output, None, None, 'CPU #0: ') + processor_brand = output.split('CPU #0: "')[1].split('"\n')[0] + cache_size = '' #_get_field(False, output, None, None, 'machdep.cpu.cache.size') + signature = output.split('Signature:')[1].split('\n')[0].strip() + # + stepping = int(signature.split('stepping ')[1].split(',')[0].strip()) + model = int(signature.split('model ')[1].split(',')[0].strip()) + family = int(signature.split('family ')[1].split(',')[0].strip()) + + # Flags + def get_subsection_flags(output): + retval = [] + for line in output.split('\n')[1:]: + if not line.startswith(' '): break + for entry in line.strip().lower().split(' '): + retval.append(entry) + return retval + + flags = get_subsection_flags(output.split('Features: ')[1]) + \ + get_subsection_flags(output.split('Extended Features (0x00000001): ')[1]) + \ + get_subsection_flags(output.split('Extended Features (0x80000001): ')[1]) + flags.sort() + + # Convert from GHz/MHz string to Hz + scale, hz_advertised = _get_hz_string_from_brand(processor_brand) + hz_actual = hz_advertised + + info = { + 'vendor_id' : vendor_id, + 'brand' : processor_brand, + + 'hz_advertised' : to_friendly_hz(hz_advertised, scale), + 'hz_actual' : to_friendly_hz(hz_actual, scale), + 'hz_advertised_raw' : to_raw_hz(hz_advertised, scale), + 'hz_actual_raw' : to_raw_hz(hz_actual, scale), + + 'l2_cache_size' : to_friendly_bytes(cache_size), 'stepping' : stepping, 'model' : model, 'family' : family, - 'processor_type' : 0, - 'extended_model' : 0, - 'extended_family' : 0, 'flags' : flags } + + info = {k: v for k, v in info.items() if v} + return info except: - return None + return {} -def get_cpu_info_from_registry(): +def _get_cpu_info_from_wmic(): + ''' + Returns the CPU info gathered from WMI. + Returns {} if not on Windows, or wmic is not installed. + ''' + + try: + # Just return {} if not Windows or there is no wmic + if not DataSource.is_windows or not DataSource.has_wmic(): + return {} + + returncode, output = DataSource.wmic_cpu() + if output == None or returncode != 0: + return {} + + # Break the list into key values pairs + value = output.split("\n") + value = [s.rstrip().split('=') for s in value if '=' in s] + value = {k: v for k, v in value if v} + + # Get the advertised MHz + processor_brand = value.get('Name') + scale_advertised, hz_advertised = _get_hz_string_from_brand(processor_brand) + + # Get the actual MHz + hz_actual = value.get('CurrentClockSpeed') + scale_actual = 6 + if hz_actual: + hz_actual = to_hz_string(hz_actual) + + # Get cache sizes + l2_cache_size = value.get('L2CacheSize') + if l2_cache_size: + l2_cache_size = l2_cache_size + ' KB' + + l3_cache_size = value.get('L3CacheSize') + if l3_cache_size: + l3_cache_size = l3_cache_size + ' KB' + + # Get family, model, and stepping + family, model, stepping = '', '', '' + description = value.get('Description') or value.get('Caption') + entries = description.split(' ') + + if 'Family' in entries and entries.index('Family') < len(entries)-1: + i = entries.index('Family') + family = int(entries[i + 1]) + + if 'Model' in entries and entries.index('Model') < len(entries)-1: + i = entries.index('Model') + model = int(entries[i + 1]) + + if 'Stepping' in entries and entries.index('Stepping') < len(entries)-1: + i = entries.index('Stepping') + stepping = int(entries[i + 1]) + + info = { + 'vendor_id' : value.get('Manufacturer'), + 'brand' : processor_brand, + + 'hz_advertised' : to_friendly_hz(hz_advertised, scale_advertised), + 'hz_actual' : to_friendly_hz(hz_actual, scale_actual), + 'hz_advertised_raw' : to_raw_hz(hz_advertised, scale_advertised), + 'hz_actual_raw' : to_raw_hz(hz_actual, scale_actual), + + 'l2_cache_size' : l2_cache_size, + 'l3_cache_size' : l3_cache_size, + + 'stepping' : stepping, + 'model' : model, + 'family' : family, + } + + info = {k: v for k, v in info.items() if v} + return info + except: + #raise # NOTE: To have this throw on error, uncomment this line + return {} + +def _get_cpu_info_from_registry(): ''' FIXME: Is missing many of the newer CPU flags like sse3 - Returns the CPU info gathered from the Windows Registry. Will return None if - not on Windows. + Returns the CPU info gathered from the Windows Registry. + Returns {} if not on Windows. ''' try: - # Just return None if not on Windows + # Just return {} if not on Windows if not DataSource.is_windows: - return None + return {} # Get the CPU name processor_brand = DataSource.winreg_processor_brand() @@ -1410,9 +1999,8 @@ def is_set(bit): flags = [k for k, v in flags.items() if v] flags.sort() - return { + info = { 'vendor_id' : vendor_id, - 'hardware' : '', 'brand' : processor_brand, 'hz_advertised' : to_friendly_hz(hz_advertised, scale), @@ -1420,50 +2008,37 @@ def is_set(bit): 'hz_advertised_raw' : to_raw_hz(hz_advertised, scale), 'hz_actual_raw' : to_raw_hz(hz_actual, 6), - 'arch' : arch, - 'bits' : bits, - 'count' : DataSource.cpu_count, - 'raw_arch_string' : raw_arch_string, - - 'l2_cache_size' : 0, - 'l2_cache_line_size' : 0, - 'l2_cache_associativity' : 0, - - 'stepping' : 0, - 'model' : 0, - 'family' : 0, - 'processor_type' : 0, - 'extended_model' : 0, - 'extended_family' : 0, 'flags' : flags } + + info = {k: v for k, v in info.items() if v} + return info except: - return None + return {} -def get_cpu_info_from_kstat(): +def _get_cpu_info_from_kstat(): ''' - Returns the CPU info gathered from isainfo and kstat. Will - return None if isainfo or kstat are not found. + Returns the CPU info gathered from isainfo and kstat. + Returns {} if isainfo or kstat are not found. ''' try: - # Just return None if there is no isainfo or kstat + # Just return {} if there is no isainfo or kstat if not DataSource.has_isainfo() or not DataSource.has_kstat(): - return None + return {} - # If isainfo fails return None + # If isainfo fails return {} returncode, flag_output = DataSource.isainfo_vb() if flag_output == None or returncode != 0: - return None + return {} - # If kstat fails return None + # If kstat fails return {} returncode, kstat = DataSource.kstat_m_cpu_info() if kstat == None or returncode != 0: - return None + return {} # Various fields vendor_id = kstat.split('\tvendor_id ')[1].split('\n')[0].strip() processor_brand = kstat.split('\tbrand ')[1].split('\n')[0].strip() - cache_size = 0 stepping = int(kstat.split('\tstepping ')[1].split('\n')[0].strip()) model = int(kstat.split('\tmodel ')[1].split('\n')[0].strip()) family = int(kstat.split('\tfamily ')[1].split('\n')[0].strip()) @@ -1481,12 +2056,8 @@ def get_cpu_info_from_kstat(): hz_actual = kstat.split('\tcurrent_clock_Hz ')[1].split('\n')[0].strip() hz_actual = to_hz_string(hz_actual) - # Get the CPU arch and bits - arch, bits = parse_arch(DataSource.raw_arch_string) - - return { + info = { 'vendor_id' : vendor_id, - 'hardware' : '', 'brand' : processor_brand, 'hz_advertised' : to_friendly_hz(hz_advertised, scale), @@ -1494,68 +2065,100 @@ def get_cpu_info_from_kstat(): 'hz_advertised_raw' : to_raw_hz(hz_advertised, scale), 'hz_actual_raw' : to_raw_hz(hz_actual, 0), - 'arch' : arch, - 'bits' : bits, - 'count' : DataSource.cpu_count, - 'raw_arch_string' : DataSource.raw_arch_string, - - 'l2_cache_size' : cache_size, - 'l2_cache_line_size' : 0, - 'l2_cache_associativity' : 0, - 'stepping' : stepping, 'model' : model, 'family' : family, - 'processor_type' : 0, - 'extended_model' : 0, - 'extended_family' : 0, 'flags' : flags } + + info = {k: v for k, v in info.items() if v} + return info except: - return None + return {} + +def CopyNewFields(info, new_info): + keys = [ + 'vendor_id', 'hardware', 'brand', 'hz_advertised', 'hz_actual', + 'hz_advertised_raw', 'hz_actual_raw', 'arch', 'bits', 'count', + 'raw_arch_string', 'l2_cache_size', 'l2_cache_line_size', + 'l2_cache_associativity', 'stepping', 'model', 'family', + 'processor_type', 'extended_model', 'extended_family', 'flags', + 'l3_cache_size', 'l1_data_cache_size', 'l1_instruction_cache_size' + ] + + for key in keys: + if new_info.get(key, None) and not info.get(key, None): + info[key] = new_info[key] + elif key == 'flags' and new_info.get('flags'): + for f in new_info['flags']: + if f not in info['flags']: info['flags'].append(f) + info['flags'].sort() def get_cpu_info(): - info = None + ''' + Returns the CPU info by using the best sources of information for your OS. + Returns {} if nothing is found. + ''' + + # Get the CPU arch and bits + arch, bits = parse_arch(DataSource.raw_arch_string) + + friendly_maxsize = { 2**31-1: '32 bit', 2**63-1: '64 bit' }.get(sys.maxsize) or 'unknown bits' + friendly_version = "{0}.{1}.{2}.{3}.{4}".format(*sys.version_info) + PYTHON_VERSION = "{0} ({1})".format(friendly_version, friendly_maxsize) + + info = { + 'python_version' : PYTHON_VERSION, + 'cpuinfo_version' : CPUINFO_VERSION, + 'arch' : arch, + 'bits' : bits, + 'count' : DataSource.cpu_count, + 'raw_arch_string' : DataSource.raw_arch_string, + } + + # Try the Windows wmic + CopyNewFields(info, _get_cpu_info_from_wmic()) # Try the Windows registry - if not info: - info = get_cpu_info_from_registry() + CopyNewFields(info, _get_cpu_info_from_registry()) # Try /proc/cpuinfo - if not info: - info = get_cpu_info_from_proc_cpuinfo() + CopyNewFields(info, _get_cpu_info_from_proc_cpuinfo()) + + # Try cpufreq-info + CopyNewFields(info, _get_cpu_info_from_cpufreq_info()) + + # Try LSCPU + CopyNewFields(info, _get_cpu_info_from_lscpu()) # Try sysctl - if not info: - info = get_cpu_info_from_sysctl() + CopyNewFields(info, _get_cpu_info_from_sysctl()) # Try kstat - if not info: - info = get_cpu_info_from_kstat() + CopyNewFields(info, _get_cpu_info_from_kstat()) # Try dmesg - if not info: - info = get_cpu_info_from_dmesg() + CopyNewFields(info, _get_cpu_info_from_dmesg()) - # Try dmesg saved during boot. - if not info: - info = get_cpu_info_from_dmesg_boot() + # Try /var/run/dmesg.boot + CopyNewFields(info, _get_cpu_info_from_cat_var_run_dmesg_boot()) + + # Try lsprop ibm,pa-features + CopyNewFields(info, _get_cpu_info_from_ibm_pa_features()) # Try sysinfo - if not info: - info = get_cpu_info_from_sysinfo() + CopyNewFields(info, _get_cpu_info_from_sysinfo()) # Try querying the CPU cpuid register - if not info: - info = get_cpu_info_from_cpuid() + CopyNewFields(info, _get_cpu_info_from_cpuid()) return info # Make sure we are running on a supported system def _check_arch(): arch, bits = parse_arch(DataSource.raw_arch_string) - if not arch in ['X86_32', 'X86_64', 'ARM_7', 'ARM_8']: - raise Exception("py-cpuinfo currently only works on X86 and some ARM CPUs.") + if not arch in ['X86_32', 'X86_64', 'ARM_7', 'ARM_8', 'PPC_64']: + raise Exception("py-cpuinfo currently only works on X86 and some PPC and ARM CPUs.") def main(): try: @@ -1566,6 +2169,8 @@ def main(): info = get_cpu_info() if info: + print('Python Version: {0}'.format(info.get('python_version', ''))) + print('Cpuinfo Version: {0}'.format(info.get('cpuinfo_version', ''))) print('Vendor ID: {0}'.format(info.get('vendor_id', ''))) print('Hardware Raw: {0}'.format(info.get('hardware', ''))) print('Brand: {0}'.format(info.get('brand', ''))) @@ -1579,10 +2184,12 @@ def main(): print('Raw Arch String: {0}'.format(info.get('raw_arch_string', ''))) + print('L1 Data Cache Size: {0}'.format(info.get('l1_data_cache_size', ''))) + print('L1 Instruction Cache Size: {0}'.format(info.get('l1_instruction_cache_size', ''))) print('L2 Cache Size: {0}'.format(info.get('l2_cache_size', ''))) print('L2 Cache Line Size: {0}'.format(info.get('l2_cache_line_size', ''))) print('L2 Cache Associativity: {0}'.format(info.get('l2_cache_associativity', ''))) - + print('L3 Cache Size: {0}'.format(info.get('l3_cache_size', ''))) print('Stepping: {0}'.format(info.get('stepping', ''))) print('Model: {0}'.format(info.get('model', ''))) print('Family: {0}'.format(info.get('family', ''))) @@ -1596,6 +2203,8 @@ def main(): if __name__ == '__main__': + from multiprocessing import freeze_support + freeze_support() main() else: _check_arch() diff --git a/setup.py b/setup.py index 1fe5e6e0..b478b360 100644 --- a/setup.py +++ b/setup.py @@ -24,8 +24,9 @@ # determine CPU support for SSE2 and AVX2 cpu_info = cpuinfo.get_cpu_info() -have_sse2 = 'sse2' in cpu_info['flags'] -have_avx2 = 'avx2' in cpu_info['flags'] +flags = cpu_info.get('flags', []) +have_sse2 = 'sse2' in flags +have_avx2 = 'avx2' in flags disable_sse2 = 'DISABLE_NUMCODECS_SSE2' in os.environ disable_avx2 = 'DISABLE_NUMCODECS_AVX2' in os.environ From 6315e58515dfa692552ef66a8a8ff542ab7bb844 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Fri, 27 Apr 2018 15:00:59 +0100 Subject: [PATCH 02/17] Changed msgpack-python to msgpack. Changed dev requirements to point to latest version on PyPI. Closes #74. --- numcodecs/msgpacks.py | 3 +-- numcodecs/tests/test_msgpacks.py | 2 +- requirements.txt | 2 +- requirements_dev.txt | 2 +- setup.py | 2 +- 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/numcodecs/msgpacks.py b/numcodecs/msgpacks.py index 07fcfd57..7d7cff99 100644 --- a/numcodecs/msgpacks.py +++ b/numcodecs/msgpacks.py @@ -28,8 +28,7 @@ class MsgPack(Codec): Notes ----- - Requires `msgpack-python `_ to be - installed. + Requires `msgpack `_ to be installed. """ diff --git a/numcodecs/tests/test_msgpacks.py b/numcodecs/tests/test_msgpacks.py index fae64279..18c9680d 100644 --- a/numcodecs/tests/test_msgpacks.py +++ b/numcodecs/tests/test_msgpacks.py @@ -9,7 +9,7 @@ try: from numcodecs.msgpacks import MsgPack except ImportError: # pragma: no cover - raise unittest.SkipTest("msgpack-python not available") + raise unittest.SkipTest("msgpack not available") from numcodecs.tests.common import (check_config, check_repr, check_encode_decode_array, diff --git a/requirements.txt b/requirements.txt index 33f83f3f..b813046c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ numpy -msgpack-python +msgpack pytest diff --git a/requirements_dev.txt b/requirements_dev.txt index 96619e74..9b1193ce 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -12,7 +12,7 @@ flake8==3.5.0 idna==2.6 mccabe==0.6.1 monotonic==1.3 -msgpack-python==0.4.8 +msgpack==0.5.6 numpy==1.13.3 packaging==16.8 pkginfo==1.4.1 diff --git a/setup.py b/setup.py index b478b360..1f37a3c6 100644 --- a/setup.py +++ b/setup.py @@ -322,7 +322,7 @@ def run_setup(with_extensions): 'numpy>=1.7', ], extras_require={ - 'msgpack': ["msgpack-python"], + 'msgpack': ["msgpack"], }, ext_modules=ext_modules, cmdclass=cmdclass, From f518cf9c8ff1374c1d2e2f2b4559339dc3ad7238 Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Fri, 27 Apr 2018 17:37:10 +0100 Subject: [PATCH 03/17] added release note --- docs/release.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/release.rst b/docs/release.rst index fea95bb5..a74a32d0 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -1,6 +1,8 @@ Release notes ============= +* Updated the msgpack dependency (by :user:`Jerome Kelleher `; :issue:`74`, :issue:`75`). + .. _release_0.5.5: 0.5.5 From cd3083e93b4a0114c830c53977632f63bdac7251 Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Fri, 27 Apr 2018 17:40:55 +0100 Subject: [PATCH 04/17] add contributor --- docs/index.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/index.rst b/docs/index.rst index 4f7f2d36..1abda830 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -86,6 +86,7 @@ documentation, code reviews, comments and/or ideas: * :user:`Francesc Alted ` * :user:`Prakhar Goel ` +* :user:`Jerome Kelleher ` * :user:`John Kirkham ` * :user:`Alistair Miles ` * :user:`Jeff Reback ` From 88bf001f66344b15bc34b2c51da852338951d24d Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Fri, 4 May 2018 11:32:14 +0100 Subject: [PATCH 05/17] Ensures that get_codec doesn't modify its argument. Closes #78. --- numcodecs/registry.py | 1 + numcodecs/tests/test_registry.py | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/numcodecs/registry.py b/numcodecs/registry.py index 64dca304..c11a1b36 100644 --- a/numcodecs/registry.py +++ b/numcodecs/registry.py @@ -29,6 +29,7 @@ def get_codec(config): Zlib(level=1) """ + config = dict(config) codec_id = config.pop('id', None) cls = codec_registry.get(codec_id, None) if cls is None: diff --git a/numcodecs/tests/test_registry.py b/numcodecs/tests/test_registry.py index 476388ba..3322e975 100644 --- a/numcodecs/tests/test_registry.py +++ b/numcodecs/tests/test_registry.py @@ -11,3 +11,11 @@ def test_registry_errors(): with pytest.raises(ValueError): get_codec({'id': 'foo'}) + + +def test_get_codec_argument(): + # Check that get_codec doesn't modify its argument. + arg = {"id": "json"} + before = dict(arg) + get_codec(arg) + assert before == arg From 8a1cd44ffadbc678b57bec0b9a584735b371ae5d Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Fri, 4 May 2018 11:39:24 +0100 Subject: [PATCH 06/17] Removed minor spam from test output. --- numcodecs/tests/common.py | 1 - 1 file changed, 1 deletion(-) diff --git a/numcodecs/tests/common.py b/numcodecs/tests/common.py index b3f080e0..76068564 100644 --- a/numcodecs/tests/common.py +++ b/numcodecs/tests/common.py @@ -241,7 +241,6 @@ def check_err_decode_object_buffer(compressor): out = np.empty(10, dtype=object) with pytest.raises(ValueError): compressor.decode(enc, out=out) - print(out[:]) def check_err_encode_object_buffer(compressor): From 4d5a248f70d3d02bcc923cc4cdd0086657b60642 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Thu, 3 May 2018 10:51:27 +0100 Subject: [PATCH 07/17] Added tests to verify handling of non-numpy inputs. --- numcodecs/tests/test_json.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/numcodecs/tests/test_json.py b/numcodecs/tests/test_json.py index de493bde..8c61eb51 100644 --- a/numcodecs/tests/test_json.py +++ b/numcodecs/tests/test_json.py @@ -54,3 +54,23 @@ def test_repr(): def test_backwards_compatibility(): check_backwards_compatibility(JSON.codec_id, arrays, codecs) + + +def test_non_numpy_inputs(): + # numpy will infer a range of different shapes and dtypes for these inputs. + # Make sure that round-tripping through encode preserves this. + data = [ + [0, 1], + [[0, 1], [2, 3]], + [[0], [1], [2, 3]], + [[[0, 0]], [[1, 1]], [[2, 3]]], + ["1"], + ["11", "11"], + ["11", "1", "1"], + [{}], + [{"key": "value"}, ["list", "of", "strings"]], + ] + for input_data in data: + for codec in codecs: + output_data = codec.decode(codec.encode(input_data)) + assert np.array_equal(np.array(input_data), output_data) From de4e32c22a686512d4ea9be0441d469f09cc1d04 Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Thu, 3 May 2018 11:59:32 +0100 Subject: [PATCH 08/17] Basic changes needed for json2 codec. --- fixture/json2/array.00.npy | Bin 0 -> 2101 bytes fixture/json2/array.01.npy | Bin 0 -> 4195 bytes fixture/json2/array.02.npy | Bin 0 -> 4185 bytes fixture/json2/array.03.npy | Bin 0 -> 4080 bytes fixture/json2/array.04.npy | Bin 0 -> 10880 bytes fixture/json2/array.05.npy | Bin 0 -> 2132 bytes fixture/json2/array.06.npy | Bin 0 -> 81680 bytes fixture/json2/array.07.npy | Bin 0 -> 2953 bytes fixture/json2/codec.00/config.json | 15 + fixture/json2/codec.00/encoded.00.dat | 1 + fixture/json2/codec.00/encoded.01.dat | 1 + fixture/json2/codec.00/encoded.02.dat | 1 + fixture/json2/codec.00/encoded.03.dat | 1 + fixture/json2/codec.00/encoded.04.dat | 1 + fixture/json2/codec.00/encoded.05.dat | 1 + fixture/json2/codec.00/encoded.06.dat | 1 + fixture/json2/codec.00/encoded.07.dat | 1 + fixture/json2/codec.01/config.json | 15 + fixture/json2/codec.01/encoded.00.dat | 906 ++++++++ fixture/json2/codec.01/encoded.01.dat | 1507 +++++++++++++ fixture/json2/codec.01/encoded.02.dat | 906 ++++++++ fixture/json2/codec.01/encoded.03.dat | 1006 +++++++++ fixture/json2/codec.01/encoded.04.dat | 906 ++++++++ fixture/json2/codec.01/encoded.05.dat | 3006 +++++++++++++++++++++++++ fixture/json2/codec.01/encoded.06.dat | 1206 ++++++++++ fixture/json2/codec.01/encoded.07.dat | 1206 ++++++++++ fixture/json2/codec.02/config.json | 15 + fixture/json2/codec.02/encoded.00.dat | 1 + fixture/json2/codec.02/encoded.01.dat | 1 + fixture/json2/codec.02/encoded.02.dat | 1 + fixture/json2/codec.02/encoded.03.dat | 1 + fixture/json2/codec.02/encoded.04.dat | 1 + fixture/json2/codec.02/encoded.05.dat | 1 + fixture/json2/codec.02/encoded.06.dat | 1 + fixture/json2/codec.02/encoded.07.dat | 1 + fixture/json2/codec.03/config.json | 15 + fixture/json2/codec.03/encoded.00.dat | 903 ++++++++ fixture/json2/codec.03/encoded.01.dat | 1503 +++++++++++++ fixture/json2/codec.03/encoded.02.dat | 903 ++++++++ fixture/json2/codec.03/encoded.03.dat | 1003 +++++++++ fixture/json2/codec.03/encoded.04.dat | 903 ++++++++ fixture/json2/codec.03/encoded.05.dat | 3003 ++++++++++++++++++++++++ fixture/json2/codec.03/encoded.06.dat | 1203 ++++++++++ fixture/json2/codec.03/encoded.07.dat | 1203 ++++++++++ numcodecs/__init__.py | 3 +- numcodecs/json.py | 49 +- numcodecs/tests/test_json.py | 42 +- 47 files changed, 21434 insertions(+), 9 deletions(-) create mode 100644 fixture/json2/array.00.npy create mode 100644 fixture/json2/array.01.npy create mode 100644 fixture/json2/array.02.npy create mode 100644 fixture/json2/array.03.npy create mode 100644 fixture/json2/array.04.npy create mode 100644 fixture/json2/array.05.npy create mode 100644 fixture/json2/array.06.npy create mode 100644 fixture/json2/array.07.npy create mode 100644 fixture/json2/codec.00/config.json create mode 100644 fixture/json2/codec.00/encoded.00.dat create mode 100644 fixture/json2/codec.00/encoded.01.dat create mode 100644 fixture/json2/codec.00/encoded.02.dat create mode 100644 fixture/json2/codec.00/encoded.03.dat create mode 100644 fixture/json2/codec.00/encoded.04.dat create mode 100644 fixture/json2/codec.00/encoded.05.dat create mode 100644 fixture/json2/codec.00/encoded.06.dat create mode 100644 fixture/json2/codec.00/encoded.07.dat create mode 100644 fixture/json2/codec.01/config.json create mode 100644 fixture/json2/codec.01/encoded.00.dat create mode 100644 fixture/json2/codec.01/encoded.01.dat create mode 100644 fixture/json2/codec.01/encoded.02.dat create mode 100644 fixture/json2/codec.01/encoded.03.dat create mode 100644 fixture/json2/codec.01/encoded.04.dat create mode 100644 fixture/json2/codec.01/encoded.05.dat create mode 100644 fixture/json2/codec.01/encoded.06.dat create mode 100644 fixture/json2/codec.01/encoded.07.dat create mode 100644 fixture/json2/codec.02/config.json create mode 100644 fixture/json2/codec.02/encoded.00.dat create mode 100644 fixture/json2/codec.02/encoded.01.dat create mode 100644 fixture/json2/codec.02/encoded.02.dat create mode 100644 fixture/json2/codec.02/encoded.03.dat create mode 100644 fixture/json2/codec.02/encoded.04.dat create mode 100644 fixture/json2/codec.02/encoded.05.dat create mode 100644 fixture/json2/codec.02/encoded.06.dat create mode 100644 fixture/json2/codec.02/encoded.07.dat create mode 100644 fixture/json2/codec.03/config.json create mode 100644 fixture/json2/codec.03/encoded.00.dat create mode 100644 fixture/json2/codec.03/encoded.01.dat create mode 100644 fixture/json2/codec.03/encoded.02.dat create mode 100644 fixture/json2/codec.03/encoded.03.dat create mode 100644 fixture/json2/codec.03/encoded.04.dat create mode 100644 fixture/json2/codec.03/encoded.05.dat create mode 100644 fixture/json2/codec.03/encoded.06.dat create mode 100644 fixture/json2/codec.03/encoded.07.dat diff --git a/fixture/json2/array.00.npy b/fixture/json2/array.00.npy new file mode 100644 index 0000000000000000000000000000000000000000..1bf53cec8cf487764de8bb04a86c3251959a0194 GIT binary patch literal 2101 zcmeH@Jx{|h5Qd!w3YhN}_G}4K6qSv#uyw#{iXtI}0Y#4EDwWbW=Qu;4NDP#UnI8;p zQ;8ozZn%5z)Qn0=YaB_=L|!UcF@*s&S2$0aEf@qXp~E63 z63$A@^R>WWh6X|?hMlmhYev;9zlPmU=mxO2g8cvv9Ab<=%q8IM$3)eIHXeo@ZJ6A^ z5kbUQ!Epd54r$&k;MDj1`yU;fMD7%v#b3{G?u4dxO@)BoM#WS@f2&^LBJHL9w9PGn TmOx9OCD0Q1cLcbC%h>w`?%oSS literal 0 HcmV?d00001 diff --git a/fixture/json2/array.01.npy b/fixture/json2/array.01.npy new file mode 100644 index 0000000000000000000000000000000000000000..7ca10eaeedb7e29a9677b67f8dbf3fbb0cdc2bf8 GIT binary patch literal 4195 zcmeH}y-ve05P+Qq3Yh=G17u5(qNu0~Vqq&1R#Q|7DGaD`99OB-#y!UwDvDsC6*Di( z25t&G#ob`vo$qe2?7QzXe(2plI;2aU{Y;iZ``4bo==-7PkCZl=7el2pxpsH?q?FcQ z=6oit9dz35(Cb`;-i!Aq^sOO^YC4-oLTMRItBDzNt@)e|wG^r-jjn{DKsHgd$Tp|o zF!Bx!F%&A3qNK7|D-_(sA%vu`l{D=>;bvT1!uArH1K3%??f~`z=JY~`xl5qGH8V4x z+0^}{VN?27aKH$2R&Y3gqkwG+F5ozhJPAr}2(ku-3gMn4!pg9-ou literal 0 HcmV?d00001 diff --git a/fixture/json2/array.02.npy b/fixture/json2/array.02.npy new file mode 100644 index 0000000000000000000000000000000000000000..904a92e4557333baa4da5740fb638bb957e84d62 GIT binary patch literal 4185 zcmeH}%}&EG41m)X284fjf!v1*{#`dqf-QXf{U)(>TuQsom>1-ZysYEz!Nj@iphPpV7UeuqvBJ1f|YVb5pQ;LTYA{jFH&`OGBl zC$>o#Ucmt)%v!-=3`ag|%PrtIilV!3Bb$ZwG@RrgPjKodj>#HH31_#xFQXb^_81Mk p1AGVg7qA~-PsF)^;}DSmu?W#{GYt`o5Q`9t5R3j{kRb022Tybfm9YFM*Q z(})o&MHn2|J~X^tSYWY$z!IGc1O^2JcIX+_CoH5#>z-llL%-H5g>(xKRS)kJ(koP+ zEpKpea8UN3fT02Zk5~~zL?RKHC`2V1(TPD!zBa}t4snS`d=ik5L?k8&Nl8X>Qjn5V zq$Uk%Nk@7D$UsIik(n%HC6H`nCkHtRA{V*ILoj*CM}7)WkU|uu2t_GIaY|5AZhTiM2TcCeFO z>}C&p*~fkkaF9bB<_JeQ#&J$?l2e@K3}-pVc`k5~OI+p(SGmS@Zg7)Z+~y8`P7Goai`c{=F7b#@0uqvl#3Ugp$w*ELQj&_)q#-ToNKXJ6 z$VetKlZC7Vl8x--ASXfOA~$&mCNKHOPXP*2h{6=1D8(pF2})9m(v+brs7?)PQj6Nup)U2PPXij#h{iO*x{lC{=Cq(Ct!PafLTF1n+7n6#I?{>GbfGKV z=uQuM(u>}N5l$ca(vSWOU?77S%n*h$jNy!6B%>J37{)S=@l0SMlbFmDrZSD`%wQ(7 zn9UsKGLQKzU?GcG%o3KejODCgC97D?8rHIo^=x1xo7l`2wz7@w>|iIm*v%gHvXA{7 z;2?)M%n^=qjN_c(B&Rsd8P0N!^IYH}m$=Lou5yj*+~6j+xXm5za*z8w;31EA%oCpS zjOV=IC9inR8{YDc_k7?Z-|#Kp@jXBABR}yozwj%+@jHL;Cx7ub|L`xL_{@KN;p^d# zNF*W?g{VX$Ix&bzEMgOfxWpqq2}npH5|f0aBqKQ~NJ%PElZLdUBRv6RAS0Q`Oct^d zNH(&QgPa7Bi`?WPn7rg8KLsdAAqrE3q7hfil%qTqs7NI$Q-!KjqdGOH zNiAwqhq~0GJ`HF{BO23$rZl5DEoezATGNIQ+R~2pgwlbIbfPm|=t?)b(}SM$qBmiL z(}%wFqdx-}$RGwYgrN*$I3pOzC`L1ev5aFp6PU;(CNqVpOk+ATn8_?=Gl#j%V?GO5 z$RZZAgrzKFIV)JnDps?GwX9=38`#JuHnWATY-2k+*vT$-vxmLxV?PHt$RQ4Mgrgke zI43yCDNb{Svz+5R7r4kJE^~#eT;n=7xXCSUbBDX!<30~~$Ri%}gr_{?IWKt0D_--4 zx4h#$ANa^Oe9L!y&ky{_PyEa;{K{|q&L8~AU;NEK{L3dk^B-UM`t(O65|N2QRH6}` z7{nwNv57-m;t`(&BqR}uNkUSRk(?ByBo(PiLt4_2o&YkCkxXPJ3t0&y8`;T0PJ+lq zZt@UJUhrl%y1;DMMMxQJxA^q!N{>LRG3!of_1n7PYBEUFuPv z1~jA*jcGztn$esVw4@cSX+sEYX-9iP=|D$1(U~rEr5oMpK~H+on=r!ZLtpyQp8*VH z5Q7=QP=+y_5sYLMqZz|k#xb4=Ok@(1nZi`2F`XIAWEQiT!(8Sup9L&r5sO*EQkJot z6|7_xt69TZ*0G)qY-AIg*}_(~v7H_4WEZ>H!(R5Wp937^5QjO!QI2t(6P)A}r#Zt} z&T*a#T;vj$xx!Vhah)67t<8 literal 0 HcmV?d00001 diff --git a/fixture/json2/array.04.npy b/fixture/json2/array.04.npy new file mode 100644 index 0000000000000000000000000000000000000000..df4419c6f06f9b76ae80fe30a6c3d4a5af47cd5b GIT binary patch literal 10880 zcmeI!u?oU45P;!x^(nGNAw$7QaqgzLba0YrQyj!d;$-Pl_`+VP@fjTcz>zd`IxhF! zd*AF1$C=d9+cxA&vKwQ!&B{7syO^HI-%l~MVZ5*XEeE~NSN{n5y;xV3TRJnC-iQ;426vNy4|6Zl`H_aw$Or@kb9!kzck}8=og#i^?IEjiW7z8cB zWJ4w*&Qi>ib%LQ17zm*dszFV^6RO6^HEg{@tq_pCDqa;GhqO4k^yf;mG&>hi|P-AZ-Q5;l}_@oS>pzBO#zZfJRY;RKn?f`%4eB sh{+ewbT;2&a7L~+8HMw>n#J|F5jV?N5-16j1WE!Wf&WH;E4T=~A5xJLi~s-t literal 0 HcmV?d00001 diff --git a/fixture/json2/array.06.npy b/fixture/json2/array.06.npy new file mode 100644 index 0000000000000000000000000000000000000000..acbc660dad7b66ae58627b3a568c6540ac2a7094 GIT binary patch literal 81680 zcmeIz&q`Er9ER~rbvv^LRIcm-WTixx%T zBHDEaZ3^18>;mfj<{a<=h1x#{Ug!UD<}*IHv$%Zy=E{*a-(C&YMq8`n!R2mnWqJ0} zV7ePT*c@+_zr0Tg&S4sqCPmUO&nnXI*wH`;m>aI%mDU9i0j&>RgANqkkXuye_NHFr2LJtM}61ci1`Z@+daqr!nyqKwvU(3)ww={|yQlYV zKg{)d&DY$n%f+lL>UY<3>w10ux?cBn*<4oVtNVUtQ`!Dw=V)v1viI5BtjmL+a=Dv* zp4|SB%P(2o|1qoQ??2DdOlv#YlWdr6W~cJz77F(tkN)|`f46r3Mr*HET04^K|8w-; zOZnG+Wz$)g)py;0H{Wx;Ecbtg-28jo&CmMo>+fTj)$982qyA3nc5hsl2c6^UiPp|% zXR^7;W!*Q(E>3Qr%;iFM*zqP0)aR++U7w}iOPATpp2KP^?4>n($sa-vd)ag7fBOHW zm)N`X{|i0rg&y`o4|}19z0kv6=wUDPuorsR3q9(`no2}n1u2u=v9n>bo3}d&1S>R^ zScKf7Uc5A5@lfysEhIIiVD6iLVM{6=`zPpRBmNCN>^r=9@Ap228D@Snjm(4PdyfOc zlJGpH5?iriw@_?-C6++3XQst0Y^<7=N(S%})@{OZo8ltjethx9!b0MD0=+>0McfD~ zMyXJ&Bo)&l$wEnI8n!H45mzmun1;=)lEOp~hFPMa4xb z2LwUL!H7J{M;$ZGScJ2$U^ENow%~jgF2vJ;lS*6GeFUmn)uyPZ|(n310 z1>;$`6i*M^t;0knleznci>K1J8C=f2ehOFO@;I;h*-V)_Mun20ni7OOEJgC@VfTZj ztHh9CQXb=Z_lZ?5+31m}sS-?iP>N`%fU%|*umoW_!n0(}Knm?{n~2e){X?{-9sPK( zK}4S5!Au^rJYg8!Fb-Ohdo|?j_v+m2IC~Pz$YGw*uQvLd zjed2vUv2iQpO0Jbe!s25tQ_L8p>xuyA0KT0+BrDsRAJ7WEJbv^`z?XWn%-@$Nf7l` z_C&RC*+h1kFe1UcEb$g@d&6yga$7rYd(+e0{phy75A4<(w_S&;G(;!q6b;h|ou)H% pmd?>Aou@wT?}L9H{CnZ^fX@Ry51c|CAaVc2K3zW-!r)pi`4<$PCSw2q literal 0 HcmV?d00001 diff --git a/fixture/json2/codec.00/config.json b/fixture/json2/codec.00/config.json new file mode 100644 index 00000000..1f76b065 --- /dev/null +++ b/fixture/json2/codec.00/config.json @@ -0,0 +1,15 @@ +{ + "allow_nan": true, + "check_circular": true, + "encoding": "utf-8", + "ensure_ascii": true, + "id": "json2", + "indent": null, + "separators": [ + ",", + ":" + ], + "skipkeys": false, + "sort_keys": true, + "strict": true +} \ No newline at end of file diff --git a/fixture/json2/codec.00/encoded.00.dat b/fixture/json2/codec.00/encoded.00.dat new file mode 100644 index 00000000..85e91743 --- /dev/null +++ b/fixture/json2/codec.00/encoded.00.dat @@ -0,0 +1 @@ +["foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","foo","bar","baz","|O",[900]] \ No newline at end of file diff --git a/fixture/json2/codec.00/encoded.01.dat b/fixture/json2/codec.00/encoded.01.dat new file mode 100644 index 00000000..0ce60323 --- /dev/null +++ b/fixture/json2/codec.00/encoded.01.dat @@ -0,0 +1 @@ +[["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],["foo","bar",NaN],"|O",[300,3]] \ No newline at end of file diff --git a/fixture/json2/codec.00/encoded.02.dat b/fixture/json2/codec.00/encoded.02.dat new file mode 100644 index 00000000..fc4fbef8 --- /dev/null +++ b/fixture/json2/codec.00/encoded.02.dat @@ -0,0 +1 @@ +["foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"foo",1.0,2,"|O",[900]] \ No newline at end of file diff --git a/fixture/json2/codec.00/encoded.03.dat b/fixture/json2/codec.00/encoded.03.dat new file mode 100644 index 00000000..d1fe35fd --- /dev/null +++ b/fixture/json2/codec.00/encoded.03.dat @@ -0,0 +1 @@ +[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268,269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335,336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402,403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469,470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536,537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603,604,605,606,607,608,609,610,611,612,613,614,615,616,617,618,619,620,621,622,623,624,625,626,627,628,629,630,631,632,633,634,635,636,637,638,639,640,641,642,643,644,645,646,647,648,649,650,651,652,653,654,655,656,657,658,659,660,661,662,663,664,665,666,667,668,669,670,671,672,673,674,675,676,677,678,679,680,681,682,683,684,685,686,687,688,689,690,691,692,693,694,695,696,697,698,699,700,701,702,703,704,705,706,707,708,709,710,711,712,713,714,715,716,717,718,719,720,721,722,723,724,725,726,727,728,729,730,731,732,733,734,735,736,737,738,739,740,741,742,743,744,745,746,747,748,749,750,751,752,753,754,755,756,757,758,759,760,761,762,763,764,765,766,767,768,769,770,771,772,773,774,775,776,777,778,779,780,781,782,783,784,785,786,787,788,789,790,791,792,793,794,795,796,797,798,799,800,801,802,803,804,805,806,807,808,809,810,811,812,813,814,815,816,817,818,819,820,821,822,823,824,825,826,827,828,829,830,831,832,833,834,835,836,837,838,839,840,841,842,843,844,845,846,847,848,849,850,851,852,853,854,855,856,857,858,859,860,861,862,863,864,865,866,867,868,869,870,871,872,873,874,875,876,877,878,879,880,881,882,883,884,885,886,887,888,889,890,891,892,893,894,895,896,897,898,899,900,901,902,903,904,905,906,907,908,909,910,911,912,913,914,915,916,917,918,919,920,921,922,923,924,925,926,927,928,929,930,931,932,933,934,935,936,937,938,939,940,941,942,943,944,945,946,947,948,949,950,951,952,953,954,955,956,957,958,959,960,961,962,963,964,965,966,967,968,969,970,971,972,973,974,975,976,977,978,979,980,981,982,983,984,985,986,987,988,989,990,991,992,993,994,995,996,997,998,999," Date: Sun, 13 May 2018 21:54:28 +0100 Subject: [PATCH 09/17] document changes/deprecation --- numcodecs/json.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/numcodecs/json.py b/numcodecs/json.py index 3331afe0..3d8d9f13 100644 --- a/numcodecs/json.py +++ b/numcodecs/json.py @@ -12,8 +12,11 @@ class JSON(Codec): - """Codec to encode data as JSON. Useful for encoding an array of Python string - objects. + """Codec to encode data as JSON. Useful for encoding an array of Python objects. + + .. versionchanged:: 0.6 + The encoding format has been changed to include the array shape in the encoded + data, which ensures that all object arrays can be correctly encoded and decoded. Examples -------- @@ -89,10 +92,15 @@ def __repr__(self): class LegacyJSON(JSON): - """Codec to encode data as JSON. Useful for encoding an array of Python string - objects. + """Deprecated JSON codec. + + .. deprecated:: 0.6.0 + This codec is maintained to enable decoding of data previously encoded, however + there may be issues with encoding and correctly decoding certain object arrays, + hence the :class:`JSON` codec should be used instead for encoding new data. See + https://github.com/zarr-developers/numcodecs/issues/76 and + https://github.com/zarr-developers/numcodecs/pull/77 for more information. - TODO: Deprecation documentation. """ codec_id = 'json' @@ -109,7 +117,6 @@ def __init__(self, encoding='utf-8', skipkeys=False, ensure_ascii=True, indent=indent, separators=separators, strict=strict) - # TODO emit warning for using a broken codec? def encode(self, buf): buf = np.asanyarray(buf) From 7eabd04c1e3ea1331d09732a8d07e15903c588de Mon Sep 17 00:00:00 2001 From: Jerome Kelleher Date: Mon, 14 May 2018 20:36:04 +0100 Subject: [PATCH 10/17] Added msgpack2 codec and LegacyMsgPack class. --- fixture/msgpack2/array.00.npy | Bin 0 -> 2149 bytes fixture/msgpack2/array.01.npy | Bin 0 -> 4243 bytes fixture/msgpack2/array.02.npy | Bin 0 -> 4233 bytes fixture/msgpack2/array.03.npy | Bin 0 -> 4128 bytes fixture/msgpack2/array.04.npy | Bin 0 -> 10928 bytes fixture/msgpack2/array.05.npy | Bin 0 -> 2180 bytes fixture/msgpack2/array.06.npy | Bin 0 -> 81728 bytes fixture/msgpack2/array.07.npy | Bin 0 -> 3001 bytes fixture/msgpack2/codec.00/config.json | 4 ++ fixture/msgpack2/codec.00/encoded.00.dat | 1 + fixture/msgpack2/codec.00/encoded.01.dat | Bin 0 -> 5411 bytes fixture/msgpack2/codec.00/encoded.02.dat | Bin 0 -> 4210 bytes fixture/msgpack2/codec.00/encoded.03.dat | Bin 0 -> 2627 bytes fixture/msgpack2/codec.00/encoded.04.dat | 1 + fixture/msgpack2/codec.00/encoded.05.dat | Bin 0 -> 8110 bytes fixture/msgpack2/codec.00/encoded.06.dat | Bin 0 -> 22812 bytes fixture/msgpack2/codec.00/encoded.07.dat | Bin 0 -> 22810 bytes numcodecs/__init__.py | 3 +- numcodecs/msgpacks.py | 48 +++++++++++++++++-- numcodecs/tests/test_msgpacks.py | 56 ++++++++++++++++++++--- 20 files changed, 103 insertions(+), 10 deletions(-) create mode 100644 fixture/msgpack2/array.00.npy create mode 100644 fixture/msgpack2/array.01.npy create mode 100644 fixture/msgpack2/array.02.npy create mode 100644 fixture/msgpack2/array.03.npy create mode 100644 fixture/msgpack2/array.04.npy create mode 100644 fixture/msgpack2/array.05.npy create mode 100644 fixture/msgpack2/array.06.npy create mode 100644 fixture/msgpack2/array.07.npy create mode 100644 fixture/msgpack2/codec.00/config.json create mode 100644 fixture/msgpack2/codec.00/encoded.00.dat create mode 100644 fixture/msgpack2/codec.00/encoded.01.dat create mode 100644 fixture/msgpack2/codec.00/encoded.02.dat create mode 100644 fixture/msgpack2/codec.00/encoded.03.dat create mode 100644 fixture/msgpack2/codec.00/encoded.04.dat create mode 100644 fixture/msgpack2/codec.00/encoded.05.dat create mode 100644 fixture/msgpack2/codec.00/encoded.06.dat create mode 100644 fixture/msgpack2/codec.00/encoded.07.dat diff --git a/fixture/msgpack2/array.00.npy b/fixture/msgpack2/array.00.npy new file mode 100644 index 0000000000000000000000000000000000000000..b34ab6788612cc538faf1502cf25c025eecbe534 GIT binary patch literal 2149 zcmeH@Jx{|h5Qd!w3YhN}_G}4K6qSv#urXjYMUjxgfFj3nl}c%xa~z~Vkr*fyGd~#I zroVud+;I2a$9u!`?LUm}9}V=3UhRaJOxo9$Js;byWlw}ulID?+315wGX;yMQFH>4@ z-8;jY7nC%`aj56S@KHEMYf*J%<>h4|4%{^D$9%p^f`tM;pdh za6k|-mT(xrkwcnyb2#>W|L#Y}I*}U%C-K)4oI0VYT@xXow^lKg(BG&RI7@qJKW%eM Upe4`}XbH3g{v83X;5_zz0egrMa{vGU literal 0 HcmV?d00001 diff --git a/fixture/msgpack2/array.01.npy b/fixture/msgpack2/array.01.npy new file mode 100644 index 0000000000000000000000000000000000000000..3b57d80782c501ed2f7d93114cf628126b006183 GIT binary patch literal 4243 zcmeH}y-ve05P+Q)3Yh=G17u5(qN=D1VqpUbt0}646b4i|j;mHmGsP?FMe|EO1}k{|bF;h^(CDRYxc-JF_x1Q?50y=txVUvcl+67z(70qFGX( zg2Tu=G{i`#M2dpSY^6|WMh+n)hK;CY_mrDyb`F~hXboX&3EM;1@tM;P9M)U_{jHgp zDz~Y-QNyMTE?|!l<}6`<2nRl^3(ny%48z-RE9-?_890hR9^u%Jnl@{!6tve`%r%_c zynb1yl}USo2HpX_1N;lv53ncVT)=UNNPt*`Xtfv z-PpAWgA3SWggHytAHqS%nsReEjH2lF+sb-jJp)JS$0Hnvv1hYJO2NrZ@5`!Im^(%T q?*QKc{srs@*b{Ls;5bAiKrBKuTu(#9BE%xZBE+IUSR@Ucrkx*nG5tpX literal 0 HcmV?d00001 diff --git a/fixture/msgpack2/array.03.npy b/fixture/msgpack2/array.03.npy new file mode 100644 index 0000000000000000000000000000000000000000..a69ba6cb9741e728aa5f5ab90e7d2fa6391a0cb8 GIT binary patch literal 4128 zcmbW(W0w#L69C|B+il#i)n?naxz%RwX4|%H+qP}H%erRW-lxCt&Nl`q4s0J9-YzV#SU_Nj&IJO40s=eq4C@mX(xY|Hu=b%}>y<*fg@>w#cM9nh zD$kZTI5;>cdr-j8fd4mQMGz5*L}a26m1smK1~K{C7@Ii6B_8ofKtd9cm?R`68OcdO zN>Y)UG^8aR=?NeM8OcOuvXGTPvXPw}MQr5Vj>K}%ZEnl^;cmUgr! zln!*H6P@WoSGv)i9`vLay$K_nKJ=v@{TaYO1~Hf+3}qO@8NoS|UJKW_S z_j$lW9`TqbJmneBdBICw@tQZh`iXCpy!G zu5_b2J?KdEMhTBSjsY%vx1eZVl``6%R1JxfsJfpGh5ioHny{ao$O*ad)Ui9 z_H%%P9O5uXILa}ObApqc;xuPC%Q?<-fs0(?GFQ0DHLi1mo800yceu+v?(=|$JmN7= zc*--L^MaSW;x%u0%RAol4d3z|-}3`M@_~>1#LxV~ul&aE{K236#ozqHzkK2||M7*d zhd&~bh)fis5{>A@ASSVhO&sD9kN6}YA&E##5|WaPHNAm8eV=s#1;W z)SxD{s7)Q}QjhvHpdpQDOcR>YjOMhUC9P;p8$xJHJK7UU2RhP;&UB$G-RMpadeV#D zgb_|3`qGd73}7IG7|alcGK}GjU?ig$%^1cqj`2)jB9oZR6s9tb>C9jzvzW~s<}#1@ zEMOsvSj-ZZvW(@dU?r)hZbx46w6?sAX&Jm4XZc+3-?@{H%a;3cnk z%^TkGj`w`Sw|vL<{J@WV;3GfrGr#aFzwtYN@F#!qH~;W2pZLsweBtZUACX8zCJIrB zMs#8jlUT$i4snS`d=ik5L?k8&Nl8X>Qjn5Vq$Uk%Nk@7D$UsIik(n%HC6H`nCkHtR zA{V*ILoj*CM}7)WkU|uu2t_GIaY|5AZhTiM2TcCeFO>}C&p*~fkkaF9bB<_JeQ#&J$?l2e@K z3}-pVc`k5~OI+p(SGmS@Zg7)Z+~y8!hIO*;+l14dqdgh-oM}{_;@TJW z4Ro6L1a8g0ICF0QVfa41$L_;!!cCOp))_KcXSbrCOHCd8PrZblb$=yjdRwF7Uo#BDxOeCq28B-Wgv4xYUn1Vsj z5==H=BH}E?JXt3gDuICz3ZWX*^gE_%lw88bJJfvGT)~zP+YT|hZ^r15H-`hy^X|X3@<2KY4#ST=963QnyM{tQy$_9|3aNzSyUv#$ tXc3bypy}k_qHq#d^NhhMxya&r+=!cHED4kZN&+Q;lE8l>z!jW@?hl@w7`y-g literal 0 HcmV?d00001 diff --git a/fixture/msgpack2/array.06.npy b/fixture/msgpack2/array.06.npy new file mode 100644 index 0000000000000000000000000000000000000000..1418a5fc0e94162273704eaa2e8d61040adccfe5 GIT binary patch literal 81728 zcmeIzyJ}Qn7)Ieg*ojz)rR)?k3PGX>f>?-!Si}Q_cxrS=M&l`wNlYWTid;c1!NNij zQiz>*V54AX=>_Qfnc0v938?kj@Mg|CGtXp^ts4v1Z{3?}8}0RAX|%RD9$f4Omln=m z7|e8oN2}xY@o?qA>Ue2X4_qBSUmNBAwdLWUmvOpJ6yz-B<6Wzwe-P+|6_9x%FPVY%yETzGn3vx@-eg@K{Fuv~ z?9=4-`&@p`>i!Q|J%8_c=4V^m%ARGzY&AQcH#b+fe|hxJKk}=!hc{c>yw}=PuK&x? ze=p@<`?MB)IqYTEq5tXs zmtJD`(*G~?uorsR3q9AnO(mkPf|SYb*x9hz&D)&>f)$!d zEJAKkFJ2n3cqsUR7LuA$F!#;Buq7=X`zPpR<6q#Zhkb`P@BQA#FvHAmrjdELbpJ^} zC<`xQDzOzSb_>PUmSYJNdv03H!p4eesbm1(#kx&6Zc|(&+>bBbSXfA0PoS6RzeQXR zDn_YLtRxlFBFREYXBxIFToG3+qL_xwtdhb+5QbTzp$?-ENDDOxDl3Yq62%sYF)&0p zBL@UQ$iav_%10eD%~*u9uVFL`=QiPd7B0lofu*k_H4y)GR9U6S3opt+Ua))<#?nGM zunFT?xD-zh+pWPwCX>1Mhl{7uw;5c{y?F*#;_^7J`qfOCI!1+(p_&qeJS;`>=u!8h zrK`k{U{W6Ac@KzHF4^d@si_i7c~FXIsDQDi7qA3jIl{AK%|Ht6ZkdSD!@UEvsvZ9P zpg}~Q;K58DvpiuK-7#5rM}lcN%2B7`>^g1d8|uAs4th1@?Dgv0>^Qp;%*bJ$(XTf8 z8;yQ-r(bRMt6z>k|>mU^Ewu=7P~&Fq#WSbHQjX7~Z*HQH}qkvy3{-00nQk;Q#;t literal 0 HcmV?d00001 diff --git a/fixture/msgpack2/codec.00/encoded.02.dat b/fixture/msgpack2/codec.00/encoded.02.dat new file mode 100644 index 0000000000000000000000000000000000000000..e1dac01cdf5d6dc64fc7e5d0977f68ddfdb5abdd GIT binary patch literal 4210 zcmcb^+_pF^KmWA-2L>=;8s&|K1jA@57|jn1qp4uDR0P%uqh-NpSumOkMq5ar=HX~r fFj^Ljrh?J3V6-e4?K6YY07+%Rq8k5+XPH|79juc0 literal 0 HcmV?d00001 diff --git a/fixture/msgpack2/codec.00/encoded.03.dat b/fixture/msgpack2/codec.00/encoded.03.dat new file mode 100644 index 0000000000000000000000000000000000000000..f671b8df35965e98080222824df31c2781974207 GIT binary patch literal 2627 zcmWO22Llj-8inD``687D4YEpuc1EO>QVMNtErq0PN+>feqDW~^qP?`I7S;DEdu3*3 z@6El>Pk0W-;R13LJmSbgM-?to^yp&6OO!n3*ixm-ls)eFawnX4(#fZkuW)L`N|mde zc6!xn)oawORlCj^XP$L--E+>ZcV7Jl=QnJ0LE|PDHf?rM^A;Cha%sy}t=qJ1*Z#5& z9XoaI(zV;=-Fx)x)%%J*eXqRg>VDVszjnaD>#o1y#+z=w<<{E<4ZeNI9e3U}blBbZ z+?)61{rNx+&j)ivj?7UxI>+Qg`EZWSNAl4em*exXd^{)Q6FD&_<&!x%r{vU}meX@a z&dgakJLlxwoR?4K(>Xt%$!BvxF3d%_IG5zoT$amoMXtJ$K~J+?BiYg?ushI4XfkqGQmps1z!V%Am67ICMNJ zhfY8zqLa|c=oC~QRY0eril`E*jH;m1(CMfus)nkg8mK0!g=(WZ=nQlwIt!hR>Y{Vd zxu_mG57kEv(D|q#YJ@I8jZqVHA!>@6p^H#+)B;_ME)D2yZx}zSbC+dZIqbpD!)E8Zeu0mI%e&`z1A6<(Epn>Q*bUnHO-H2{N zH=|q7t>`v12n|NJqao-HbSJtC4MoGy-RK^4FS-xij~+n7(Sv9N8i_`s(P#{M2tAC( zqDRo9XdD`k9z&0#3Frwl5luo*qRD6qnu?~O>1YO;iDseMXbzf-=Aoz1(`Y_=20e=w zpoM4=T8x&UrDz#ij#i+RXcc-6tww9mTC@(WM;p*av*x*iCVC6Kjov};qJ!u?^gj9keTY6nAEQI) z6Z9$i41JEiKwqN6=qvO!`UZWAzC+)mAJC8JC-gJ=1^tSCL%*Xx(4Xip^f&qk{fqv? z1+cJj;DY!Fd?YS}kHUp<5nL1>jf>&pxCAbVkHN>{Qn)lOgUjOM@bS1DJ^`PIPr@hT zQ*e1)0iTL1;!3zOu7Xd)r{k))8m^9O;F`D=u8r&9Gw_-CEPOVui_gL5;(GWzTpu^U z=i`RB5xxL7#!c{rxG8RiFT%}n3w$xY1Ye3<;#RmdZiCz6cDOyh40pgCaVOjvcfnn8 zH+(tnj(gyqxEJn?ufTn9UwkFL3SW);;cIYzd@UY;2jc7S_4o#SBfbgWjBmlW;@j{b zJQ&}Ohu}N#o%k+16c599<9qPE_&$6;egF^058@GcBp!uF<1zRl{4gGiAHk2}adDFY@Td4Q{5k#t ze~Ay{ukhFS8~iQ)4u6k-z(3-j@Xz=c{44$q|BnB_f8xLJ-}oQ=FaD1xK!`9AL_y*R z;z*(paTHOQC_)q^jwXr`#fcI`N#YpdSfUhBnkYk*C5|JGC(02g5GN8R5hoL;5ao#q z#HmC@q7qS=s6w1ZoK93Fsu9(R8bnQ^7Ezn1L!3dJNt{KTP1GgMAOH5HAz25U&!i5w8<(5N{H15pNUk5bqKPiT8;2 fi4TYmiI0eni9^IE#HYk(#OL!G4ygP8hA;jHSt+b{ literal 0 HcmV?d00001 diff --git a/fixture/msgpack2/codec.00/encoded.04.dat b/fixture/msgpack2/codec.00/encoded.04.dat new file mode 100644 index 00000000..b4740367 --- /dev/null +++ b/fixture/msgpack2/codec.00/encoded.04.dat @@ -0,0 +1 @@ +���foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�foo�bar�baz�)zD; zQQp~C`Izd|q_)s{=V2Ol!#Q{k{-JabhyoFoh=_=Yh=_=Yh=_=Yh=_=Yh=_=Yh=_=Y Uh=_=Yh=_>jFA;4+wVYvk0E5K9;Q#;t literal 0 HcmV?d00001 diff --git a/fixture/msgpack2/codec.00/encoded.06.dat b/fixture/msgpack2/codec.00/encoded.06.dat new file mode 100644 index 0000000000000000000000000000000000000000..853aafc66896335a57d69595b61fdec427731732 GIT binary patch literal 22812 zcmeI(Jx{_w9LI47UdY|<;^gAw)XBxD#I3OxC@D07(%E9%x){{O1rmq@Ac|sI9ztSg z%WfP>igDyEi1;F0zDpm*rtRJR@BjIIW(K{tre<5ZC{{~`Eqj_Oh~ubLwhUF0-9uG& zs}*r#n}+Oa$`nOiH?5*BU$bUOoaLj2Eu8#fG!r>`|Ha|Je0%KtEaVk>UC1DnR@+YaQl7Z5B^T8iNBmSzS`5(#OrIy zvZCj0;hI*|%*o!VQPy2sR9xjK`LQLF@H`0~lAxW0wasyTd!Ga^zt_Qa68bzN?~!kn z|B*3=ah17aGhH%2vd&>G$9k1DIduoM74;)EGJ75D=dfqRzMM~uOpQ#9OfP_b2R$A7 zR`llRKhk5RkBkSvJm5O8EBG7?63z-QhY7?zVmtAx7+oAP)*3TUE`aO<`3^EHS3!;)di euw+;=EE$#zONJ%GlKqz@YwaEF>|VWR9)AGqKjm@& literal 0 HcmV?d00001 diff --git a/fixture/msgpack2/codec.00/encoded.07.dat b/fixture/msgpack2/codec.00/encoded.07.dat new file mode 100644 index 0000000000000000000000000000000000000000..6a738658c91dc526c5b5594e327df3a195794246 GIT binary patch literal 22810 zcmeI(Jx{_w9LI47UdY|v!LphGQ#HIxd@z>>9c#_TsyWZR)b@?COel zTowC{Wy+4BEm2a+vP(+79#~~@REQgna0~O{ROGGs=*H@87nNJot`b|08@J^ssV9RZ zN}j~*ESb*EMKYWPtIKgRkb}&xXSI6fw==()`MpJW{e2bm|4z$su$VNyT9f72?-|;* z+mv-0-llg$hXS> z$e6>p%G|M@E}0)$=dhMzy~>)Lx`Wz^`jHx$y$<$s*t23^E}%xHMy5ul7eK#*o(_F0 zdUNz2>9NvB#sgp;a2?ncd=3T)XN8x;1mYgCo%mIZE)E%MjhQDGK=y%r2N@P}I%JW^ zTamXSZ$;jUycL!VONJ%Gl3~fPWLPpR8I}x7h9$$2Vac#$STZabmJCaVCBu?o$*^Qt bGAtRE3`>S3!;)di{>zfxp6pz_=N^9mBwOV| literal 0 HcmV?d00001 diff --git a/numcodecs/__init__.py b/numcodecs/__init__.py index 9e5fc318..bd0a5461 100644 --- a/numcodecs/__init__.py +++ b/numcodecs/__init__.py @@ -87,8 +87,9 @@ register_codec(Pickle) try: - from numcodecs.msgpacks import MsgPack + from numcodecs.msgpacks import MsgPack, LegacyMsgPack register_codec(MsgPack) + register_codec(LegacyMsgPack) except ImportError: # pragma: no cover pass diff --git a/numcodecs/msgpacks.py b/numcodecs/msgpacks.py index 7d7cff99..c78076d2 100644 --- a/numcodecs/msgpacks.py +++ b/numcodecs/msgpacks.py @@ -7,12 +7,17 @@ from .abc import Codec +from .compat import buffer_tobytes class MsgPack(Codec): """Codec to encode data as msgpacked bytes. Useful for encoding an array of Python string objects. + .. versionchanged:: 0.6 + The encoding format has been changed to include the array shape in the encoded + data, which ensures that all object arrays can be correctly encoded and decoded. + Examples -------- >>> import numcodecs @@ -32,20 +37,23 @@ class MsgPack(Codec): """ - codec_id = 'msgpack' + codec_id = 'msgpack2' def __init__(self, encoding='utf-8'): self.encoding = encoding def encode(self, buf): - buf = np.asarray(buf) + buf = np.asanyarray(buf) items = buf.tolist() items.append(buf.dtype.str) + items.append(buf.shape) return msgpack.packb(items, encoding=self.encoding) def decode(self, buf, out=None): + buf = buffer_tobytes(buf) items = msgpack.unpackb(buf, encoding=self.encoding) - dec = np.array(items[:-1], dtype=items[-1]) + dec = np.empty(items[-1], dtype=items[-2]) + dec[:] = items[:-2] if out is not None: np.copyto(out, dec) return out @@ -58,3 +66,37 @@ def get_config(self): def __repr__(self): return 'MsgPack(encoding=%r)' % self.encoding + + +class LegacyMsgPack(MsgPack): + """Deprecated MsgPack codec. + + .. deprecated:: 0.6.0 + This codec is maintained to enable decoding of data previously encoded, however + there may be issues with encoding and correctly decoding certain object arrays, + hence the :class:`MsgPack` codec should be used instead for encoding new data. + See https://github.com/zarr-developers/numcodecs/issues/76 and + https://github.com/zarr-developers/numcodecs/pull/77 for more information. + + """ + + codec_id = 'msgpack' + + def encode(self, buf): + buf = np.asanyarray(buf) + items = buf.tolist() + items.append(buf.dtype.str) + return msgpack.packb(items, encoding=self.encoding) + + def decode(self, buf, out=None): + buf = buffer_tobytes(buf) + items = msgpack.unpackb(buf, encoding=self.encoding) + dec = np.array(items[:-1], dtype=items[-1]) + if out is not None: + np.copyto(out, dec) + return out + else: + return dec + + def __repr__(self): + return 'LegacyMsgPack(encoding=%r)' % self.encoding diff --git a/numcodecs/tests/test_msgpacks.py b/numcodecs/tests/test_msgpacks.py index 18c9680d..fe677bf6 100644 --- a/numcodecs/tests/test_msgpacks.py +++ b/numcodecs/tests/test_msgpacks.py @@ -1,13 +1,15 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, division import unittest +import itertools import numpy as np try: - from numcodecs.msgpacks import MsgPack + from numcodecs.msgpacks import LegacyMsgPack, MsgPack + codecs = [LegacyMsgPack(), MsgPack()] except ImportError: # pragma: no cover raise unittest.SkipTest("msgpack not available") @@ -33,20 +35,62 @@ def test_encode_decode(): - for arr in arrays: - codec = MsgPack() + for arr, codec in itertools.product(arrays, codecs): check_encode_decode_array(arr, codec) def test_config(): - codec = MsgPack() - check_config(codec) + for codec in codecs: + check_config(codec) def test_repr(): check_repr("MsgPack(encoding='utf-8')") check_repr("MsgPack(encoding='ascii')") + check_repr("LegacyMsgPack(encoding='utf-8')") + check_repr("LegacyMsgPack(encoding='ascii')") def test_backwards_compatibility(): - check_backwards_compatibility(MsgPack.codec_id, arrays, [MsgPack()]) + for codec in codecs: + check_backwards_compatibility(codec.codec_id, arrays, [codec]) + + +def test_non_numpy_inputs(): + # numpy will infer a range of different shapes and dtypes for these inputs. + # Make sure that round-tripping through encode preserves this. + data = [ + [0, 1], + [[0, 1], [2, 3]], + [[0], [1], [2, 3]], + [[[0, 0]], [[1, 1]], [[2, 3]]], + ["1"], + ["11", "11"], + ["11", "1", "1"], + [{}], + [{"key": "value"}, ["list", "of", "strings"]], + ] + for input_data in data: + for codec in codecs: + output_data = codec.decode(codec.encode(input_data)) + assert np.array_equal(np.array(input_data), output_data) + + +def test_legacy_codec_broken(): + # Simplest demonstration of why the MsgPack codec needed to be changed. + # The LegacyMsgPack codec didn't include shape information in the serialised + # bytes, which gave different shapes in the input and output under certain + # circumstances. + a = np.empty(2, dtype=object) + a[0] = [0, 1] + a[1] = [2, 3] + codec = LegacyMsgPack() + b = codec.decode(codec.encode(a)) + assert a.shape == (2,) + assert b.shape == (2, 2) + assert not np.array_equal(a, b) + + # Now show that the MsgPack codec handles this case properly. + codec = MsgPack() + b = codec.decode(codec.encode(a)) + assert np.array_equal(a, b) From cfd05828267ff7c1bfdc99c72474984afa035d5a Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Tue, 15 May 2018 09:49:17 +0100 Subject: [PATCH 11/17] generalise docstring --- numcodecs/msgpacks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numcodecs/msgpacks.py b/numcodecs/msgpacks.py index c78076d2..bd195cbd 100644 --- a/numcodecs/msgpacks.py +++ b/numcodecs/msgpacks.py @@ -12,7 +12,7 @@ class MsgPack(Codec): """Codec to encode data as msgpacked bytes. Useful for encoding an array of Python - string objects. + objects. .. versionchanged:: 0.6 The encoding format has been changed to include the array shape in the encoded From af0990f4156e3779822b52d216d22c8e0754d63c Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Tue, 15 May 2018 09:55:13 +0100 Subject: [PATCH 12/17] added release notes --- docs/release.rst | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/docs/release.rst b/docs/release.rst index a74a32d0..b22190ee 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -1,7 +1,22 @@ Release notes ============= -* Updated the msgpack dependency (by :user:`Jerome Kelleher `; :issue:`74`, :issue:`75`). +.. _release_0.6.0: + +0.6.0 (in development) +---------------------- + +* The encoding format used by the :class:`JSON` and :class:`MsgPack` codecs has been + changed to resolve an issue with correctly encoding and decoding some object arrays. + Now the encoded data includes the original shape of the array, which enables the + correct shape to be restored on decoding. The previous encoding format is still + supported, so that any data encoded using a previous version of numcodecs can still be + read. Thus no changes to user code and applications should be required, other + than upgrading numcodecs. By :user:`Jerome Kelleher `; :issue:`74`, + :issue:`75`. + +* Updated the msgpack dependency (by :user:`Jerome Kelleher `; + :issue:`74`, :issue:`75`). .. _release_0.5.5: From 13bcd5b590e96fde4967d6e5f6f16fcc746afc8a Mon Sep 17 00:00:00 2001 From: Anand Date: Wed, 23 May 2018 19:22:40 +0200 Subject: [PATCH 13/17] Updating docs/release.rst --- docs/release.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/release.rst b/docs/release.rst index b22190ee..eecc88ca 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -18,6 +18,8 @@ Release notes * Updated the msgpack dependency (by :user:`Jerome Kelleher `; :issue:`74`, :issue:`75`). +* Added support for ppc64le architecture by updating `cpuinfo.py` from upstream (by :user:`Anand S `) + .. _release_0.5.5: 0.5.5 From b82596b7378064e42e2f1d7eecdcb7253762472e Mon Sep 17 00:00:00 2001 From: Anand Date: Wed, 24 Oct 2018 18:31:02 +0200 Subject: [PATCH 14/17] Specifying VS2017 --- appveyor.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index 59f45c36..409cef1c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,3 +1,5 @@ +image: Visual Studio 2017 + branches: only: - master From 9492de085d812008999437ba9c284d926a20d222 Mon Sep 17 00:00:00 2001 From: Anand Date: Wed, 24 Oct 2018 18:38:26 +0200 Subject: [PATCH 15/17] Revert "Specifying VS2017" This reverts commit b82596b7378064e42e2f1d7eecdcb7253762472e. --- appveyor.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 409cef1c..59f45c36 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,5 +1,3 @@ -image: Visual Studio 2017 - branches: only: - master From 290062045b6f723972a4d5dfa0e1aef9f9c9d34d Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Tue, 27 Nov 2018 23:55:05 +0000 Subject: [PATCH 16/17] disable __AVX2__ macros on windows PY27 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 19a5eb9e..9a19ed3e 100644 --- a/setup.py +++ b/setup.py @@ -105,7 +105,7 @@ def blosc_extension(): info('compiling Blosc extension with AVX2 support') extra_compile_args.append('-DSHUFFLE_AVX2_ENABLED') blosc_sources += [f for f in glob('c-blosc/blosc/*.c') if 'avx2' in f] - if os.name == 'nt': + if os.name == 'nt' and not PY2: define_macros += [('__AVX2__', 1)] else: info('compiling Blosc extension without AVX2 support') From 928398a1fd086da43a34bdcff5fa473f802c67ee Mon Sep 17 00:00:00 2001 From: Alistair Miles Date: Wed, 28 Nov 2018 00:35:55 +0000 Subject: [PATCH 17/17] Better fix for no AVX2 on windows PY27 --- setup.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 9a19ed3e..448031eb 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,9 @@ have_avx2 = 'avx2' in flags disable_sse2 = 'DISABLE_NUMCODECS_SSE2' in os.environ disable_avx2 = 'DISABLE_NUMCODECS_AVX2' in os.environ - +if PY2 and os.name == 'nt': + # force no AVX2 on windows PY27 + disable_avx2 = True # setup common compile arguments have_cflags = 'CFLAGS' in os.environ @@ -105,7 +107,7 @@ def blosc_extension(): info('compiling Blosc extension with AVX2 support') extra_compile_args.append('-DSHUFFLE_AVX2_ENABLED') blosc_sources += [f for f in glob('c-blosc/blosc/*.c') if 'avx2' in f] - if os.name == 'nt' and not PY2: + if os.name == 'nt': define_macros += [('__AVX2__', 1)] else: info('compiling Blosc extension without AVX2 support')