From b0d39387520ad7be8a57d065745b8b9c9a94f9e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Fri, 10 Jan 2020 12:43:55 +0200 Subject: [PATCH 1/2] Add debugging for initial heap sizes --- tests/core/test_emmalloc_trim.cpp | 4 ++++ tests/core/test_emmalloc_trim.txt | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/tests/core/test_emmalloc_trim.cpp b/tests/core/test_emmalloc_trim.cpp index dc426bb780bcb..513bf95f32649 100644 --- a/tests/core/test_emmalloc_trim.cpp +++ b/tests/core/test_emmalloc_trim.cpp @@ -6,6 +6,10 @@ int main() { printf("heap size: %zu\n", emscripten_get_heap_size()); + printf("dynamic heap 0: %zu\n", emmalloc_dynamic_heap_size()); + printf("free dynamic memory 0: %zu\n", emmalloc_free_dynamic_memory()); + printf("unclaimed heap memory 0: %zu\n", emmalloc_unclaimed_heap_memory()); + printf("sbrk 0: %p\n", sbrk(0)); void *ptr = malloc(32*1024*1024); void *ptr2 = malloc(4*1024*1024); diff --git a/tests/core/test_emmalloc_trim.txt b/tests/core/test_emmalloc_trim.txt index 099fdd0f47375..60774b2b419d1 100644 --- a/tests/core/test_emmalloc_trim.txt +++ b/tests/core/test_emmalloc_trim.txt @@ -1,4 +1,8 @@ heap size: 134217728 +dynamic heap 0: 48 +free dynamic memory 0: 8 +unclaimed heap memory 0: 2142170240 +sbrk 0: 0x501380 1 dynamic heap 1: 37748880 free dynamic memory 1: 84 From cdb0fab80b0ad92948877e059a3661bdc3b7a97e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jukka=20Jyl=C3=A4nki?= Date: Fri, 10 Jan 2020 14:10:48 +0200 Subject: [PATCH 2/2] Fix emmalloc tests to have a 4k slop for varying memory heap sizes. Skip emmalloc tests in LSan and ASan which do not support custom memory allocators. --- system/lib/emmalloc.cpp | 4 + tests/core/test_emmalloc.cpp | 105 +++--------------- .../core/test_emmalloc_memory_statistics.cpp | 7 +- .../core/test_emmalloc_memory_statistics.txt | 2 +- tests/core/test_emmalloc_trim.cpp | 61 +++++----- tests/core/test_emmalloc_trim.txt | 50 ++++----- tests/test_core.py | 9 +- 7 files changed, 90 insertions(+), 148 deletions(-) diff --git a/system/lib/emmalloc.cpp b/system/lib/emmalloc.cpp index 417ae3f335adc..883b195cd8ff2 100644 --- a/system/lib/emmalloc.cpp +++ b/system/lib/emmalloc.cpp @@ -1027,6 +1027,10 @@ static int acquire_and_attempt_region_resize(Region *region, size_t size) void *emmalloc_aligned_realloc(void *ptr, size_t alignment, size_t size) { +#ifdef EMMALLOC_DEBUG_LOG + MAIN_THREAD_ASYNC_EM_ASM(console.log('aligned_realloc(ptr=' + $0.toString(16) + ', alignment=' + $1 + ', size=' + $2), ptr, alignment, size); +#endif + if (!ptr) return emmalloc_memalign(alignment, size); diff --git a/tests/core/test_emmalloc.cpp b/tests/core/test_emmalloc.cpp index 4f86f2f25bc5f..7e0ce62207d13 100644 --- a/tests/core/test_emmalloc.cpp +++ b/tests/core/test_emmalloc.cpp @@ -16,30 +16,12 @@ extern "C" void emmalloc_blank_slate_from_orbit(); -// Test emmalloc internals, but through the external interface. We expect -// very specific outputs here based on the internals, this test would not -// pass in another malloc. - -void* check_where_we_would_malloc(size_t size) { - void* temp = malloc(size); - free(temp); - return temp; -} - -void check_where_we_would_malloc(size_t size, void* expected) { - void* temp = malloc(size); - assert(temp == expected); - free(temp); -} - void stage(const char* name) { EM_ASM({ out('\n>> ' + UTF8ToString($0) + '\n'); }, name); } -const size_t ALLOCATION_UNIT = 8; - void basics() { stage("basics"); stage("allocate 0"); @@ -55,83 +37,30 @@ void basics() { stage("allocate 10"); assert(second == first); void* third = malloc(10); - assert(size_t(third) == size_t(first) + ((100 + ALLOCATION_UNIT - 1)&(-ALLOCATION_UNIT)) + ALLOCATION_UNIT); // allocation units are multiples of ALLOCATION_UNIT + assert(!emmalloc_validate_memory_regions()); stage("allocate 10 more"); void* four = malloc(10); - assert(size_t(four) == size_t(third) + (2*ALLOCATION_UNIT) + ALLOCATION_UNIT); // payload (10 = 2 allocation units) and metadata + assert(!emmalloc_validate_memory_regions()); stage("free the first"); free(second); - stage("several temp alloc/frees"); - // we reuse the first area, despite stuff later. - for (int i = 0; i < 4; i++) { - check_where_we_would_malloc(100, first); - } stage("free all"); free(third); free(four); - stage("allocate various sizes to see they all start at the start"); - for (int i = 1; i < 1500; i++) { - check_where_we_would_malloc(i, first); - } -} - -void blank_slate() { - stage("blank_slate"); - emmalloc_blank_slate_from_orbit(); - void* ptr = malloc(0); - free(ptr); - for (int i = 0; i < 3; i++) { - void* two = malloc(0); - assert(two == ptr); - free(two); - } - for (int i = 0; i < 3; i++) { - emmalloc_blank_slate_from_orbit(); - void* two = malloc(0); - // emmalloc_blank_slate_from_orbit clears out the free list without freeing the blocks on it - // Effectively, memory is leaked and we do not expect the pointers to be the same - assert(two != ptr); - free(two); - } + assert(!emmalloc_validate_memory_regions()); } void previous_sbrk() { stage("previous_sbrk"); emmalloc_blank_slate_from_orbit(); void* old = sbrk(0); - assert((size_t)old % ALLOCATION_UNIT == 0); + assert((size_t)old % 4 == 0); sbrk(3); // unalign things void* other = malloc(10); free(other); assert(other != old); } -void min_alloc() { - stage("min_alloc"); - emmalloc_blank_slate_from_orbit(); - void* start = check_where_we_would_malloc(1); - for (int i = 1; i < 100; i++) { - void* temp = malloc(i); - void* expected = (char*)start + ALLOCATION_UNIT + ALLOCATION_UNIT * ((i + ALLOCATION_UNIT - 1) / ALLOCATION_UNIT); - check_where_we_would_malloc(1, expected); - free(temp); - } -} - -void space_at_end() { - stage("space_at_end"); - emmalloc_blank_slate_from_orbit(); - void* start = check_where_we_would_malloc(1); - for (int i = 1; i < 50; i++) { - for (int j = 1; j < 50; j++) { - void* temp = malloc(i); - free(temp); - check_where_we_would_malloc(j, start); - } - } -} - -void calloc() { +void test_calloc() { stage("calloc"); emmalloc_blank_slate_from_orbit(); char* ptr = (char*)malloc(10); @@ -142,7 +71,7 @@ void calloc() { assert(ptr[0] == 0); } -void realloc() { +void test_realloc() { stage("realloc0"); emmalloc_blank_slate_from_orbit(); for (int i = 0; i < 2; i++) { @@ -168,18 +97,16 @@ void realloc() { } stage("realloc1"); emmalloc_blank_slate_from_orbit(); - { - // realloc of NULL is like malloc - void* ptr = check_where_we_would_malloc(10); - assert(realloc(NULL, 10) == ptr); - } + + // realloc of NULL is like malloc + assert(realloc(NULL, 10) != 0); + stage("realloc2"); emmalloc_blank_slate_from_orbit(); { // realloc to 0 is like free void* ptr = malloc(10); assert(realloc(ptr, 0) == NULL); - assert(check_where_we_would_malloc(10) == ptr); } stage("realloc3"); emmalloc_blank_slate_from_orbit(); @@ -230,7 +157,7 @@ void aligned() { void randoms() { stage("randoms"); emmalloc_blank_slate_from_orbit(); - void* start = check_where_we_would_malloc(10); + void* start = malloc(10); const int N = 1000; const int BINS = 128; void* bins[BINS]; @@ -285,20 +212,16 @@ void randoms() { for (int i = 0; i < BINS; i++) { if (bins[i]) free(bins[i]); } - // it's all freed, should be a blank slate - assert(check_where_we_would_malloc(10) == start); + assert(!emmalloc_validate_memory_regions()); } int main() { stage("beginning"); basics(); - blank_slate(); previous_sbrk(); - min_alloc(); - space_at_end(); - calloc(); - realloc(); + test_calloc(); + test_realloc(); aligned(); randoms(); diff --git a/tests/core/test_emmalloc_memory_statistics.cpp b/tests/core/test_emmalloc_memory_statistics.cpp index a8ca147fda9dd..4de5a5d0f5de4 100644 --- a/tests/core/test_emmalloc_memory_statistics.cpp +++ b/tests/core/test_emmalloc_memory_statistics.cpp @@ -1,6 +1,11 @@ #include #include +template +T round_to_4k(T val){ + return (T)(((size_t)val + 4095) & ~4095); +} + int main() { void *ptr = malloc(32*1024*1024); @@ -21,5 +26,5 @@ int main() for(int i = 0; i < 32; ++i) printf("%zu ", freeMemorySizeMap[i]); printf("\n"); - printf("%zu\n", emmalloc_unclaimed_heap_memory()); + printf("%zu\n", round_to_4k(emmalloc_unclaimed_heap_memory())); } diff --git a/tests/core/test_emmalloc_memory_statistics.txt b/tests/core/test_emmalloc_memory_statistics.txt index 557ff305ba36c..0daf8d72f0209 100644 --- a/tests/core/test_emmalloc_memory_statistics.txt +++ b/tests/core/test_emmalloc_memory_statistics.txt @@ -4,4 +4,4 @@ 4210892 3 0 0 0 0 0 0 0 1 0 0 0 0 0 0 1 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 -21997632 +21999616 diff --git a/tests/core/test_emmalloc_trim.cpp b/tests/core/test_emmalloc_trim.cpp index 513bf95f32649..4d21406135c63 100644 --- a/tests/core/test_emmalloc_trim.cpp +++ b/tests/core/test_emmalloc_trim.cpp @@ -3,55 +3,60 @@ #include #include +template +T round_to_4k(T val){ + return (T)(((size_t)val + 4095) & ~4095); +} + int main() { printf("heap size: %zu\n", emscripten_get_heap_size()); - printf("dynamic heap 0: %zu\n", emmalloc_dynamic_heap_size()); - printf("free dynamic memory 0: %zu\n", emmalloc_free_dynamic_memory()); - printf("unclaimed heap memory 0: %zu\n", emmalloc_unclaimed_heap_memory()); - printf("sbrk 0: %p\n", sbrk(0)); + printf("dynamic heap 0: %zu\n", round_to_4k(emmalloc_dynamic_heap_size())); + printf("free dynamic memory 0: %zu\n", round_to_4k(emmalloc_free_dynamic_memory())); + printf("unclaimed heap memory 0: %zu\n", round_to_4k(emmalloc_unclaimed_heap_memory())); + printf("sbrk 0: %p\n", round_to_4k(sbrk(0))); void *ptr = malloc(32*1024*1024); void *ptr2 = malloc(4*1024*1024); printf("%d\n", (int)(ptr && ptr2)); - printf("dynamic heap 1: %zu\n", emmalloc_dynamic_heap_size()); - printf("free dynamic memory 1: %zu\n", emmalloc_free_dynamic_memory()); - printf("unclaimed heap memory 1: %zu\n", emmalloc_unclaimed_heap_memory()); - printf("sbrk 1: %p\n", sbrk(0)); + printf("dynamic heap 1: %zu\n", round_to_4k(emmalloc_dynamic_heap_size())); + printf("free dynamic memory 1: %zu\n", round_to_4k(emmalloc_free_dynamic_memory())); + printf("unclaimed heap memory 1: %zu\n", round_to_4k(emmalloc_unclaimed_heap_memory())); + printf("sbrk 1: %p\n", round_to_4k(sbrk(0))); int success = emmalloc_trim(0); printf("1st trim: %d\n", success); - printf("dynamic heap 1: %zu\n", emmalloc_dynamic_heap_size()); - printf("free dynamic memory 1: %zu\n", emmalloc_free_dynamic_memory()); - printf("unclaimed heap memory 1: %zu\n", emmalloc_unclaimed_heap_memory()); - printf("sbrk 1: %p\n", sbrk(0)); + printf("dynamic heap 1: %zu\n", round_to_4k(emmalloc_dynamic_heap_size())); + printf("free dynamic memory 1: %zu\n", round_to_4k(emmalloc_free_dynamic_memory())); + printf("unclaimed heap memory 1: %zu\n", round_to_4k(emmalloc_unclaimed_heap_memory())); + printf("sbrk 1: %p\n", round_to_4k(sbrk(0))); success = emmalloc_trim(0); printf("2nd trim: %d\n", success); - printf("dynamic heap 2: %zu\n", emmalloc_dynamic_heap_size()); - printf("free dynamic memory 2: %zu\n", emmalloc_free_dynamic_memory()); - printf("unclaimed heap memory 2: %zu\n", emmalloc_unclaimed_heap_memory()); - printf("sbrk 2: %p\n", sbrk(0)); + printf("dynamic heap 2: %zu\n", round_to_4k(emmalloc_dynamic_heap_size())); + printf("free dynamic memory 2: %zu\n", round_to_4k(emmalloc_free_dynamic_memory())); + printf("unclaimed heap memory 2: %zu\n", round_to_4k(emmalloc_unclaimed_heap_memory())); + printf("sbrk 2: %p\n", round_to_4k(sbrk(0))); free(ptr2); success = emmalloc_trim(100000); printf("3rd trim: %d\n", success); - printf("dynamic heap 3: %zu\n", emmalloc_dynamic_heap_size()); - printf("free dynamic memory 3: %zu\n", emmalloc_free_dynamic_memory()); - printf("unclaimed heap memory 3: %zu\n", emmalloc_unclaimed_heap_memory()); - printf("sbrk 3: %p\n", sbrk(0)); + printf("dynamic heap 3: %zu\n", round_to_4k(emmalloc_dynamic_heap_size())); + printf("free dynamic memory 3: %zu\n", round_to_4k(emmalloc_free_dynamic_memory())); + printf("unclaimed heap memory 3: %zu\n", round_to_4k(emmalloc_unclaimed_heap_memory())); + printf("sbrk 3: %p\n", round_to_4k(sbrk(0))); success = emmalloc_trim(100000); printf("4th trim: %d\n", success); - printf("dynamic heap 4: %zu\n", emmalloc_dynamic_heap_size()); - printf("free dynamic memory 4: %zu\n", emmalloc_free_dynamic_memory()); - printf("unclaimed heap memory 4: %zu\n", emmalloc_unclaimed_heap_memory()); - printf("sbrk 4: %p\n", sbrk(0)); + printf("dynamic heap 4: %zu\n", round_to_4k(emmalloc_dynamic_heap_size())); + printf("free dynamic memory 4: %zu\n", round_to_4k(emmalloc_free_dynamic_memory())); + printf("unclaimed heap memory 4: %zu\n", round_to_4k(emmalloc_unclaimed_heap_memory())); + printf("sbrk 4: %p\n", round_to_4k(sbrk(0))); success = emmalloc_trim(0); printf("5th trim: %d\n", success); - printf("dynamic heap 5: %zu\n", emmalloc_dynamic_heap_size()); - printf("free dynamic memory 5: %zu\n", emmalloc_free_dynamic_memory()); - printf("unclaimed heap memory 5: %zu\n", emmalloc_unclaimed_heap_memory()); - printf("sbrk 5: %p\n", sbrk(0)); + printf("dynamic heap 5: %zu\n", round_to_4k(emmalloc_dynamic_heap_size())); + printf("free dynamic memory 5: %zu\n", round_to_4k(emmalloc_free_dynamic_memory())); + printf("unclaimed heap memory 5: %zu\n", round_to_4k(emmalloc_unclaimed_heap_memory())); + printf("sbrk 5: %p\n", round_to_4k(sbrk(0))); } diff --git a/tests/core/test_emmalloc_trim.txt b/tests/core/test_emmalloc_trim.txt index 60774b2b419d1..2d42011a80ec2 100644 --- a/tests/core/test_emmalloc_trim.txt +++ b/tests/core/test_emmalloc_trim.txt @@ -1,35 +1,35 @@ heap size: 134217728 -dynamic heap 0: 48 -free dynamic memory 0: 8 -unclaimed heap memory 0: 2142170240 -sbrk 0: 0x501380 +dynamic heap 0: 4096 +free dynamic memory 0: 4096 +unclaimed heap memory 0: 2142171136 +sbrk 0: 0x502000 1 -dynamic heap 1: 37748880 -free dynamic memory 1: 84 -unclaimed heap memory 1: 2104421504 -sbrk 1: 0x2901380 +dynamic heap 1: 37752832 +free dynamic memory 1: 4096 +unclaimed heap memory 1: 2104422400 +sbrk 1: 0x2902000 1st trim: 1 -dynamic heap 1: 37748788 +dynamic heap 1: 37752832 free dynamic memory 1: 0 -unclaimed heap memory 1: 2104421596 -sbrk 1: 0x2901324 +unclaimed heap memory 1: 2104422400 +sbrk 1: 0x2902000 2nd trim: 0 -dynamic heap 2: 37748788 +dynamic heap 2: 37752832 free dynamic memory 2: 0 -unclaimed heap memory 2: 2104421596 -sbrk 2: 0x2901324 +unclaimed heap memory 2: 2104422400 +sbrk 2: 0x2902000 3rd trim: 1 -dynamic heap 3: 33654492 -free dynamic memory 3: 100008 -unclaimed heap memory 3: 2108515892 -sbrk 3: 0x25199cc +dynamic heap 3: 33656832 +free dynamic memory 3: 102400 +unclaimed heap memory 3: 2108518400 +sbrk 3: 0x251a000 4th trim: 0 -dynamic heap 4: 33654492 -free dynamic memory 4: 100008 -unclaimed heap memory 4: 2108515892 -sbrk 4: 0x25199cc +dynamic heap 4: 33656832 +free dynamic memory 4: 102400 +unclaimed heap memory 4: 2108518400 +sbrk 4: 0x251a000 5th trim: 1 -dynamic heap 5: 33554476 +dynamic heap 5: 33558528 free dynamic memory 5: 0 -unclaimed heap memory 5: 2108615908 -sbrk 5: 0x250131c \ No newline at end of file +unclaimed heap memory 5: 2108616704 +sbrk 5: 0x2502000 \ No newline at end of file diff --git a/tests/test_core.py b/tests/test_core.py index 762226616d25b..531ab4074e2a9 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -897,6 +897,7 @@ def test_mallocstruct(self): self.do_run(self.gen_struct_src.replace('{{gen_struct}}', '(S*)malloc(sizeof(S))').replace('{{del_struct}}', 'free'), '*51,62*') @no_asan('ASan does not support custom memory allocators') + @no_lsan('LSan does not support custom memory allocators') @parameterized({ 'normal': [], 'debug': ['-DEMMALLOC_DEBUG'], @@ -914,6 +915,8 @@ def test_emmalloc(self, *args): open(path_from_root('tests', 'core', 'test_emmalloc.cpp')).read(), open(path_from_root('tests', 'core', 'test_emmalloc.txt')).read()) + @no_asan('ASan does not support custom memory allocators') + @no_lsan('LSan does not support custom memory allocators') def test_emmalloc_usable_size(self, *args): self.set_setting('MALLOC', 'emmalloc') self.emcc_args += list(args) @@ -922,7 +925,8 @@ def test_emmalloc_usable_size(self, *args): @no_fastcomp('this feature works in fastcomp, but test outputs are sensitive to wasm backend') @no_optimize('output is sensitive to optimization flags, so only test unoptimized builds') - @no_wasm2js('output is specific to wasm debug builds only') + @no_asan('ASan does not support custom memory allocators') + @no_lsan('LSan does not support custom memory allocators') def test_emmalloc_memory_statistics(self, *args): self.set_setting('MALLOC', 'emmalloc') @@ -932,7 +936,8 @@ def test_emmalloc_memory_statistics(self, *args): @no_fastcomp('this feature works in fastcomp, but test outputs are sensitive to wasm backend') @no_optimize('output is sensitive to optimization flags, so only test unoptimized builds') - @no_wasm2js('output is specific to wasm debug builds only') + @no_asan('ASan does not support custom memory allocators') + @no_lsan('LSan does not support custom memory allocators') def test_emmalloc_trim(self, *args): self.set_setting('MALLOC', 'emmalloc') self.emcc_args += ['-s', 'TOTAL_MEMORY=128MB', '-s', 'ALLOW_MEMORY_GROWTH=1'] + list(args)