From b9f4044af43111969f794ad29f4829159641bcef Mon Sep 17 00:00:00 2001 From: Sam Clegg Date: Thu, 22 Feb 2024 14:19:39 -0800 Subject: [PATCH] [test] Simplify test_openjpeg. NFC This change simplifies the test in 2 ways: 1. Remove the "extra testing" part of the test that tries to run the test again if `ALLOW_MEMORY_GROWTH` is set. The only test modes that have `ALLOW_MEMORY_GROWTH` are the santizier modes and it seems odd for an individual test to decide to disable this. 2. Remove the alternate `is_sanitizing` method of running the test. I believe this was only needed becuase of (1), and so can be safely removed. By removing this complexity this test now has less nesting and is more directly readable and more similar to other tests. --- test/test_core.py | 201 +++++++++++++++++++++------------------------- 1 file changed, 90 insertions(+), 111 deletions(-) diff --git a/test/test_core.py b/test/test_core.py index b696cb6a97148..3f21eb5efd94e 100644 --- a/test/test_core.py +++ b/test/test_core.py @@ -10,7 +10,6 @@ import random import re import shutil -import sys import time import unittest from pathlib import Path @@ -6807,117 +6806,97 @@ def test_poppler(self): @no_wasm64('MEMORY64 does not yet support SJLJ') @is_slow_test def test_openjpeg(self): - def do_test_openjpeg(): - def line_splitter(data): - out = '' - counter = 0 - - for ch in data: - out += ch - if ch == ' ' and counter > 60: - out += '\n' - counter = 0 - else: - counter += 1 - - return out - - # remove -g, so we have one test without it by default - self.emcc_args = [x for x in self.emcc_args if x != '-g'] - - original_j2k = test_file('openjpeg/syntensity_lobby_s.j2k') - image_bytes = list(bytearray(read_binary(original_j2k))) - create_file('pre.js', """ - Module.preRun = () => FS.createDataFile('/', 'image.j2k', %s, true, false, false); - Module.postRun = () => { - out('Data: ' + JSON.stringify(Array.from(FS.readFile('image.raw')))); - }; - """ % line_splitter(str(image_bytes))) - - # ensure libpng is built so that openjpeg's configure step can detect it. - # If we don't do this then we don't know what the state of the cache will be - # and this test would different non-deterministic results based on, for example, - # what other tests had previously run. - builder_cmd = [EMBUILDER, 'build', 'libpng'] - if self.get_setting('MEMORY64'): - builder_cmd.append('--wasm64') - self.emcc_args.append('-Wno-pointer-to-int-cast') - self.run_process(builder_cmd) - lib = self.get_library('third_party/openjpeg', - [Path('codec/CMakeFiles/j2k_to_image.dir/index.c.o'), - Path('codec/CMakeFiles/j2k_to_image.dir/convert.c.o'), - Path('codec/CMakeFiles/j2k_to_image.dir/__/common/color.c.o'), - Path('bin/libopenjpeg.a')], - configure=['cmake', '.'], - # configure_args=['--enable-tiff=no', '--enable-jp3d=no', '--enable-png=no'], - make_args=[]) # no -j 2, since parallel builds can fail - - # We use doubles in JS, so we get slightly different values than native code. So we - # check our output by comparing the average pixel difference - def image_compare(output): - # Get the image generated by JS, from the JSON.stringify'd array - m = re.search(r'\[[\d, -]*\]', output) - self.assertIsNotNone(m, 'Failed to find proper image output in: ' + output) - # Evaluate the output as a python array - js_data = eval(m.group(0)) - - js_data = [x if x >= 0 else 256 + x for x in js_data] # Our output may be signed, so unsign it - - # Get the correct output - true_data = bytearray(read_binary(test_file('openjpeg/syntensity_lobby_s.raw'))) - - # Compare them - self.assertEqual(len(js_data), len(true_data)) - num = len(js_data) - diff_total = js_total = true_total = 0 - for i in range(num): - js_total += js_data[i] - true_total += true_data[i] - diff_total += abs(js_data[i] - true_data[i]) - js_mean = js_total / float(num) - true_mean = true_total / float(num) - diff_mean = diff_total / float(num) - - image_mean = 83.265 - # print '[image stats:', js_mean, image_mean, true_mean, diff_mean, num, ']' - assert abs(js_mean - image_mean) < 0.01, [js_mean, image_mean] - assert abs(true_mean - image_mean) < 0.01, [true_mean, image_mean] - assert diff_mean < 0.01, diff_mean - - return output - - self.emcc_args += ['--minify=0'] # to compare the versions - self.emcc_args += ['--pre-js', 'pre.js'] - - def do_test(): - self.do_runf('third_party/openjpeg/codec/j2k_to_image.c', - 'Successfully generated', # The real test for valid output is in image_compare - args='-i image.j2k -o image.raw'.split(), - emcc_args=['-sUSE_LIBPNG'], - libraries=lib, - includes=[test_file('third_party/openjpeg/libopenjpeg'), - test_file('third_party/openjpeg/codec'), - test_file('third_party/openjpeg/common'), - Path(self.get_build_dir(), 'third_party/openjpeg')], - output_nicerizer=image_compare) - - do_test() - - # extra testing - if self.get_setting('ALLOW_MEMORY_GROWTH') == 1: - print('no memory growth', file=sys.stderr) - self.set_setting('ALLOW_MEMORY_GROWTH', 0) - do_test() + def line_splitter(data): + out = '' + counter = 0 + + for ch in data: + out += ch + if ch == ' ' and counter > 60: + out += '\n' + counter = 0 + else: + counter += 1 - if is_sanitizing(self.emcc_args): - # In ASan mode we need a large initial memory (or else wasm-ld fails). - # The OpenJPEG CMake will build several executables (which we need parts - # of in our testing, see above), so we must enable the flag for them all. - with env_modify({'EMCC_CFLAGS': '-sINITIAL_MEMORY=300MB'}): - self.emcc_args.append('-Wno-unused-command-line-argument') - do_test_openjpeg() - else: - do_test_openjpeg() + return out + + # remove -g, so we have one test without it by default + self.emcc_args = [x for x in self.emcc_args if x != '-g'] + + original_j2k = test_file('openjpeg/syntensity_lobby_s.j2k') + image_bytes = list(bytearray(read_binary(original_j2k))) + create_file('pre.js', """ + Module.preRun = () => FS.createDataFile('/', 'image.j2k', %s, true, false, false); + Module.postRun = () => { + out('Data: ' + JSON.stringify(Array.from(FS.readFile('image.raw')))); + }; + """ % line_splitter(str(image_bytes))) + + # ensure libpng is built so that openjpeg's configure step can detect it. + # If we don't do this then we don't know what the state of the cache will be + # and this test would different non-deterministic results based on, for example, + # what other tests had previously run. + builder_cmd = [EMBUILDER, 'build', 'libpng'] + if self.get_setting('MEMORY64'): + builder_cmd.append('--wasm64') + self.emcc_args.append('-Wno-pointer-to-int-cast') + self.run_process(builder_cmd) + lib = self.get_library('third_party/openjpeg', + [Path('codec/CMakeFiles/j2k_to_image.dir/index.c.o'), + Path('codec/CMakeFiles/j2k_to_image.dir/convert.c.o'), + Path('codec/CMakeFiles/j2k_to_image.dir/__/common/color.c.o'), + Path('bin/libopenjpeg.a')], + configure=['cmake', '.'], + # configure_args=['--enable-tiff=no', '--enable-jp3d=no', '--enable-png=no'], + make_args=[]) # no -j 2, since parallel builds can fail + + # We use doubles in JS, so we get slightly different values than native code. So we + # check our output by comparing the average pixel difference + def image_compare(output): + # Get the image generated by JS, from the JSON.stringify'd array + m = re.search(r'\[[\d, -]*\]', output) + + self.assertIsNotNone(m, 'Failed to find proper image output in: ' + output) + # Evaluate the output as a python array + js_data = eval(m.group(0)) + + js_data = [x if x >= 0 else 256 + x for x in js_data] # Our output may be signed, so unsign it + # Get the correct output + true_data = bytearray(read_binary(test_file('openjpeg/syntensity_lobby_s.raw'))) + + # Compare them + self.assertEqual(len(js_data), len(true_data)) + num = len(js_data) + diff_total = js_total = true_total = 0 + for i in range(num): + js_total += js_data[i] + true_total += true_data[i] + diff_total += abs(js_data[i] - true_data[i]) + js_mean = js_total / float(num) + true_mean = true_total / float(num) + diff_mean = diff_total / float(num) + + image_mean = 83.265 + # print '[image stats:', js_mean, image_mean, true_mean, diff_mean, num, ']' + assert abs(js_mean - image_mean) < 0.01, [js_mean, image_mean] + assert abs(true_mean - image_mean) < 0.01, [true_mean, image_mean] + assert diff_mean < 0.01, diff_mean + + return output + + self.emcc_args += ['--minify=0'] # to compare the versions + self.emcc_args += ['--pre-js', 'pre.js'] + + self.do_runf('third_party/openjpeg/codec/j2k_to_image.c', + 'Successfully generated', # The real test for valid output is in image_compare + args='-i image.j2k -o image.raw'.split(), + emcc_args=['-sUSE_LIBPNG'], + libraries=lib, + includes=[test_file('third_party/openjpeg/libopenjpeg'), + test_file('third_party/openjpeg/codec'), + test_file('third_party/openjpeg/common'), + Path(self.get_build_dir(), 'third_party/openjpeg')], + output_nicerizer=image_compare) @also_with_standalone_wasm(impure=True) @no_asan('autodebug logging interferes with asan')