diff --git a/vm/ByteCodeTranslator/src/cn1_globals.h b/vm/ByteCodeTranslator/src/cn1_globals.h index a8adb55c183..5c9278af9d1 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.h +++ b/vm/ByteCodeTranslator/src/cn1_globals.h @@ -1412,6 +1412,25 @@ typedef struct CN1BibopPage { // idempotent across parallel markers) int gcGraceEpoch; // upper bound on survivor epochs as of the last // full walk (GC-thread only) + JAVA_BOOLEAN gcMajorSpliced; // pulled from a PARTIAL pool by the major sweep, so + // its slots are a one-off deep-sweep sample rather + // than the steady-state retirement sample the + // adaptive trigger is calibrated on (see + // cn1BibopAdaptAfterSweep). Transient: set at the + // splice, cleared as the sweep reaches the page + JAVA_BOOLEAN gcPageReusableAdvice; // the release used MADV_FREE_REUSABLE (Darwin), so + // the slot region is marked reusable and MUST be + // restored with MADV_FREE_REUSE before anything is + // allocated into it. FALSE when the release used an + // advice with no pairing (MADV_FREE, MADV_DONTNEED), + // which is what makes an EXPECTED reuse rejection + // distinguishable from a genuine failure to restore + JAVA_BOOLEAN gcPageReleased; // the slot region has been handed back to the OS + // (madvise) and must be re-acquired before use. + // Only ever set on a page that is unreachable + // from every pool, then published with the page + // onto bibopReleasedPool under bibopMutex, so + // no lock-free reader can observe it in flight #ifdef CN1_GRACE_AUDIT int gcAuditSnapshot; // QA builds only: bumpIndex at mark start. // Relaxed __atomic access everywhere -- the diff --git a/vm/ByteCodeTranslator/src/cn1_globals.m b/vm/ByteCodeTranslator/src/cn1_globals.m index d0284ce9dd6..07a928dc1be 100644 --- a/vm/ByteCodeTranslator/src/cn1_globals.m +++ b/vm/ByteCodeTranslator/src/cn1_globals.m @@ -29,6 +29,15 @@ #include "cn1_globals.h" #include #include // clock_gettime: paces the low-memory allocation throttle +#ifndef _WIN32 +#include // getpagesize: sizes the BiBOP page-release window +#include // madvise: hands surplus empty BiBOP pages back to the OS +#include +#endif +#if defined(__APPLE__) +#include +#include +#endif #include "java_lang_Class.h" #include "java_lang_Object.h" #include "java_lang_Boolean.h" @@ -2261,6 +2270,19 @@ JAVA_INT java_lang_System_identityHashCode___java_lang_Object_R_int(CODENAME_ONE _Atomic int bibopGcEpoch = 1; _Atomic int bibopBypassGeneration[CN1_BIBOP_NUM_CLASSES]; static long bibopCycleAllocatedBytes = 0; +// LEGACY bytes charged to the cycle that is starting -- the twin of +// bibopCycleAllocatedBytes for allocations above CN1_BIBOP_MAX_OBJECT. The +// exchange result used to be discarded, which made a large-array workload look +// QUIET to the major-sweep test below however hard it was allocating (its bytes +// never reach bibopBytesSinceGc). That is the one shape where splicing every +// partial page into every sweep is worst: heavy legacy churn driving the cycles +// while a large BiBOP survivor set supplies the pages to walk -- precisely the +// issue-5425 workload. GC-thread only, published at cycle begin. +// long long, NOT long: cn1LegacyBytesSinceGc is long long for the same reason -- +// on the Windows LLP64 target long is 32 bits, so a collector that falls more +// than 2GB of legacy allocation behind would truncate this to a negative value +// and the quiet-cycle test below would read a furiously allocating app as idle. +static long long legacyCycleAllocatedBytes = 0; static long bibopLastCycleOccupiedBytes = 0; static long bibopLastCycleLiveBytes = 0; static long bibopLastCycleReclaimedBytes = 0; @@ -2323,6 +2345,17 @@ static void cn1BibopFormatPage(CN1BibopPage* p, int ci) { p->freeList = 0; p->freeCount = 0; p->owned = JAVA_FALSE; + // Page-release state. Both MUST be initialized here: a page from + // cn1BibopRawPage is indeterminate memory, and cn1BibopTrimFreePool READS + // gcPageReleased before anything has written it. A stale nonzero value would + // make the trim skip the madvise, mark the page released anyway and file it + // under bibopReleasedPool -- the release silently doing nothing for that page + // -- while a stale gcMajorSpliced would drop an ordinary page out of the + // adaptive-trigger statistics. Reformatting a recycled page also lands here, + // where both are already false, so the writes are idempotent. + p->gcPageReleased = JAVA_FALSE; + p->gcPageReusableAdvice = JAVA_FALSE; + p->gcMajorSpliced = JAVA_FALSE; #ifdef CN1_GC_VERIFY // QA: a recycled page still holds the DEAD previous occupants' headers, so a // reference that dangles into it would resolve to a plausible-looking object @@ -2378,7 +2411,8 @@ void cn1BibopBeginGcCycle(void) { // Same atomic-exchange idiom as the BiBOP reset above: a racing fetch_add // lands either before the swap (covered by the cycle that is starting) or // after it (charged to the next cycle) -- never dropped. - (void)atomic_exchange_explicit(&cn1LegacyBytesSinceGc, 0, memory_order_acq_rel); + legacyCycleAllocatedBytes = atomic_exchange_explicit(&cn1LegacyBytesSinceGc, 0, + memory_order_acq_rel); // Latch AFTER the counter: an allocator racing between the two exchanges // still sees the old latch and skips, so the fresh latch can never be // consumed by bytes just charged to the cycle that is starting. @@ -2451,12 +2485,489 @@ void cn1BibopBeginGcCycle(void) { #endif } +// ===================== PAGE RELEASE (issue 5537) ===================== +// A swept-empty page used to stay resident forever: it went to bibopFreePool and +// BiBOP had no munmap/madvise/free path at all. Because pages are ALSO +// size-class segregated, that memory was not merely idle, it was unusable by +// anything except a future block of CN1_BIBOP_MAX_OBJECT bytes or less -- a +// large array, an image buffer or a native texture could not touch it. So a +// transient small-object peak permanently subtracted its own size from the +// process budget, which on iOS is a jetsam ceiling of roughly 1.4GB rather than +// a desktop's many gigabytes. BibopPageFloorIntegrationTest measures it: after +// holding then dropping 192MB of small objects and forcing six collection +// cycles, allocating a 192MB texture set cost the full 192MB again, while the +// identical allocation over a legacy-freed hole cost nothing. +// +// The fix hands the slot region of surplus empty pages back to the OS. Three +// properties keep it cheap and safe: +// +// - ONLY fully-empty pages, and only the ones beyond a warm cache of +// CN1_BIBOP_FREE_POOL_KEEP. Steady-state churn cycles pages through the warm +// cache and never calls madvise at all; a workload whose small-object demand +// SHRINKS is the only one that pays, which is exactly the 5537 shape. +// - The page HEADER stays resident. Only the slot region is released, starting +// at the first system-page boundary at or after sizeof(CN1BibopPage), so the +// pool links, the class index and the bump cursor survive. BiBOP pages are +// CN1_BIBOP_PAGE_SIZE-aligned and that is a multiple of every system page +// size we run on, so the page base is always system-page-aligned. +// - Reads of a released region cannot fault -- the mapping is still there, they +// return zero or stale bytes. That matters because the conservative root +// resolver can still probe such a page from a stale stack word. It rejects +// what it finds either way: a zeroed slot has __heapPosition 0, which is +// neither CN1_BIBOP_HEAP_POS nor CN1_BIBOP_ADOPTED. Nothing live can be lost +// because a page only reaches this path with liveCount == 0, which the sweep +// established AFTER the mark that the conservative scan feeds. +#ifndef CN1_BIBOP_FREE_POOL_KEEP +#define CN1_BIBOP_FREE_POOL_KEEP 64 /* 64 * 64KB = 4MB kept warm, never released */ +#endif +// Bound on the madvise work one sweep may do, so a collapse from a huge pool +// cannot turn a single cycle into a syscall storm. The remainder is released by +// the following sweeps. +#ifndef CN1_BIBOP_RELEASE_PER_SWEEP +#define CN1_BIBOP_RELEASE_PER_SWEEP 1024 /* up to 64MB per cycle */ +#endif +// A cycle that allocated less than this is treated as QUIET: the app is not +// churning, so the major sweep below is both cheap (no mutator contending for +// the pages) and exactly what is wanted (a burst has just ended and its memory +// should go back). A quarter of the base trigger is comfortably below any +// allocation-driven cycle, which by construction crosses the whole trigger. +#ifndef CN1_BIBOP_MAJOR_SWEEP_QUIET_BYTES +#define CN1_BIBOP_MAJOR_SWEEP_QUIET_BYTES (CN1_BIBOP_GC_TRIGGER_BYTES / 4) +#endif +// Backstop cadence for an app that never goes quiet, so a long-running churn +// still eventually returns pages. Every cycle would make the sweep O(all pages), +// which is the regression issue 5425 fixed. +#ifndef CN1_BIBOP_MAJOR_SWEEP_CYCLES +#define CN1_BIBOP_MAJOR_SWEEP_CYCLES 16 +#endif +// Cycles since the last major sweep. GC-thread only. +static int bibopCyclesSinceMajorSweep = 0; + +// Env-gated tracer, same pattern as CN1_LOG_LOWMEM_PARKS: one line per sweep +// that actually released pages. Costs a cached getenv when disabled. +static _Atomic int cn1PageReleaseTrace = -1; +static int cn1PageReleaseTraceOn(void) { + int on = atomic_load_explicit(&cn1PageReleaseTrace, memory_order_relaxed); + if(on < 0) { + on = getenv("CN1_LOG_PAGE_RELEASE") ? 1 : 0; + atomic_store_explicit(&cn1PageReleaseTrace, on, memory_order_relaxed); + } + return on; +} + +// Last errno from a rejected MADV_FREE_REUSABLE, surfaced by the tracer. Only +// MADV_FREE_REUSABLE decrements phys_footprint; the MADV_FREE fallback leaves +// the pages charged to the process, so a nonzero value here means the release +// ran but bought nothing. +static int cn1PageReleaseReusableErrno = 0; +// CN1_BIBOP_FAIL_REUSE forces every MADV_FREE_REUSE to report failure. That path +// is otherwise unreachable -- the call does not fail in practice -- so the code +// that has to cope with a page which cannot be restored would never run outside +// a test. Deliberately NOT part of the CN1_GC_FAULT family: those live under +// CN1_GC_VERIFY, and page release is disabled in verifier builds, so a fault +// declared there could never fire. +// CN1_BIBOP_FAIL_REUSABLE= makes the first n MADV_FREE_REUSABLE calls report +// failure so the MADV_FREE fallback is taken, then behaves normally. A count +// rather than a switch because the interesting behaviour is what happens AFTER +// the transient clears: pages released through the fallback are still charged to +// the process and have to be upgraded by a later trim. +static _Atomic int cn1ReusableFailBudget = -1; +static int cn1ReusableFailInjectTake(void) { + int n = atomic_load_explicit(&cn1ReusableFailBudget, memory_order_relaxed); + if(n < 0) { + const char* e = getenv("CN1_BIBOP_FAIL_REUSABLE"); + n = (e != 0) ? atoi(e) : 0; + if(n < 0) n = 0; + atomic_store_explicit(&cn1ReusableFailBudget, n, memory_order_relaxed); + } + if(n == 0) { + return 0; + } + atomic_store_explicit(&cn1ReusableFailBudget, n - 1, memory_order_relaxed); + return 1; +} + +static _Atomic int cn1ReuseFailInject = -1; +static int cn1ReuseFailInjectOn(void) { + int on = atomic_load_explicit(&cn1ReuseFailInject, memory_order_relaxed); + if(on < 0) { + on = getenv("CN1_BIBOP_FAIL_REUSE") ? 1 : 0; + atomic_store_explicit(&cn1ReuseFailInject, on, memory_order_relaxed); + } + return on; +} + +// Last errno from a rejected MADV_FREE_REUSE on a page that WAS marked reusable. +// Distinct from the release errno above: this one means a page could not be +// taken back out of the reusable state and was therefore not handed out. +static int cn1PageReuseFailErrno = 0; + +// Empty pages whose slot region has been given back to the OS. Kept OFF +// bibopFreePool so the acquire path always prefers a warm page and only pays the +// re-acquire plus refault when no warm page is left. bibopMutex. +static CN1BibopPage* bibopReleasedPool = 0; + +// Released pages whose MADV_FREE_REUSE was rejected, so they cannot be handed to +// the allocator yet. Kept OFF bibopReleasedPool so one unrestorable page cannot +// stand in front of a stocked pool; consulted only when that pool is empty, and +// then at most one per acquisition. +// +// A FIFO, with an explicit tail, and that detail is the whole point. Pushing a +// failure back at the HEAD means the next acquisition pops the same page, fails +// again, and puts it back -- so one permanently unrestorable page is retried +// forever while every other parked page behind it is never reached again. That +// is the same defect this pool was introduced to fix, one level down. Rotating +// the failure to the tail gives round-robin retry: each acquisition tries a +// different page, and a page that becomes restorable is eventually reached. +// bibopMutex. +static CN1BibopPage* bibopReuseFailedPool = 0; +static CN1BibopPage* bibopReuseFailedTail = 0; + +// Pages sitting in bibopReleasedPool that only got the MADV_FREE fallback, so +// they are still charged to phys_footprint and want the reusable advice retried. +// Lets the upgrade pass skip its walk entirely in the normal case, which is +// every case: the fallback is only taken when MADV_FREE_REUSABLE is rejected, +// which does not happen in ordinary operation. +static _Atomic long bibopFallbackPageCount = 0; + +// Park a page whose restore was rejected at the TAIL of the retry pool. +static void cn1BibopParkReuseFailure(CN1BibopPage* p) { + p->nextPool = 0; + if(bibopReuseFailedTail != 0) { + bibopReuseFailedTail->nextPool = p; + } else { + bibopReuseFailedPool = p; + } + bibopReuseFailedTail = p; +} + +// How many released pages one acquisition may try to restore before giving up +// and allocating a fresh page. Bounds the syscalls a run of rejections can cost +// while still stepping past a bad page rather than stalling on it. +#ifndef CN1_BIBOP_REUSE_ATTEMPTS +#define CN1_BIBOP_REUSE_ATTEMPTS 4 +#endif + +#if defined(CN1_GC_INSTRUMENT) && !defined(CN1_BIBOP_NO_PAGE_RELEASE) +static _Atomic long cn1BibopPagesReleased = 0; +static _Atomic long cn1BibopPagesReacquired = 0; +#endif + +// Byte offset within a page at which the releasable slot region begins, or 0 if +// releasing is impossible on this configuration (system page so large that the +// header plus one system page would not fit). +static size_t cn1BibopReleaseOffset(void) { +#if defined(CN1_BIBOP_NO_PAGE_RELEASE) || defined(_WIN32) + return 0; +#elif defined(CN1_GC_VERIFY) + // The heap verifier works by INSPECTING memory the allocator has logically + // freed: cn1BibopFormatPage poisons every recycled slot and the verifier + // classifies a reference that lands on one as a violation. Handing those + // pages back to the OS erases that evidence -- they fault back in as zeroes, + // the conservative resolver rejects a zeroed slot on __heapPosition, and a + // genuine dangling reference reads as "no such object" instead of being + // reported. GcHeapIntegrityIntegrationTest catches this directly: with page + // release on, its deliberately re-injected grace-pass defect (issue 5425) + // stops being detected and the gate goes inert. QA builds therefore keep + // pages resident; shipping builds, which is where footprint matters, do not + // define CN1_GC_VERIFY. + return 0; +#else + static size_t cached = (size_t)-1; + if(cached == (size_t)-1) { + size_t ps = (size_t)getpagesize(); + if(ps == 0 || (ps & (ps - 1)) != 0) { + cached = 0; + } else { + size_t hdr = (sizeof(CN1BibopPage) + ps - 1) & ~(ps - 1); + cached = (hdr + ps > (size_t)CN1_BIBOP_PAGE_SIZE) ? 0 : hdr; + } + } + return cached; +#endif +} + +// Hand a fully-empty page's slot region back to the OS. Returns whether the +// advice was actually accepted: madvise can fail (a transient EAGAIN from +// Linux MADV_DONTNEED, or a range Darwin refuses), and a page whose memory was +// NOT handed back must not be recorded as released -- it would be filed under +// bibopReleasedPool and never retried, so the footprint would stay up forever +// with nothing to show that anything went wrong. The caller keeps a rejected +// page in bibopFreePool so the next sweep tries it again. +// The caller must have made the page unreachable from every pool first. +static JAVA_BOOLEAN cn1BibopReleasePageMemory(CN1BibopPage* p) { +#if !defined(CN1_BIBOP_NO_PAGE_RELEASE) && !defined(_WIN32) + size_t off = cn1BibopReleaseOffset(); + if(off == 0) { + return JAVA_FALSE; + } + void* addr = (char*)p + off; + size_t len = (size_t)CN1_BIBOP_PAGE_SIZE - off; + JAVA_BOOLEAN ok = JAVA_FALSE; +#if defined(__APPLE__) + // MADV_FREE_REUSABLE is what libmalloc uses to return large blocks, and + // unlike plain MADV_FREE it decrements phys_footprint immediately -- which + // is the figure iOS jetsam meters, so it is the one that has to move. + // MADV_FREE is kept as a fallback: it still lets the kernel take the pages + // under pressure, just without moving the accounting. Pairing MADV_FREE_REUSE + // with a range that only got MADV_FREE is harmless (it is rejected and there + // is no accounting to restore), so one released flag covers both. + if(!cn1ReusableFailInjectTake() && madvise(addr, len, MADV_FREE_REUSABLE) == 0) { + ok = JAVA_TRUE; + p->gcPageReusableAdvice = JAVA_TRUE; // must be restored before reuse + } else { + cn1PageReleaseReusableErrno = errno; + // FALLBACK. MADV_FREE lets the kernel take the pages under pressure but + // does NOT reduce phys_footprint, so this page is still charged to the + // process -- it is released in the sense that matters to the allocator + // but not in the sense this whole feature exists for. It is left with + // gcPageReusableAdvice clear, which is what cn1BibopUpgradeFallbackPages + // uses to find it and retry the reusable advice on a later trim. + ok = (madvise(addr, len, MADV_FREE) == 0) ? JAVA_TRUE : JAVA_FALSE; + p->gcPageReusableAdvice = JAVA_FALSE; // no pairing to restore + if(ok) { + atomic_fetch_add_explicit(&bibopFallbackPageCount, 1, memory_order_relaxed); + } + } +#elif defined(MADV_DONTNEED) + // Linux: drops the pages and re-faults them as zero, which is exactly the + // contract the acquire-path format expects. + ok = (madvise(addr, len, MADV_DONTNEED) == 0) ? JAVA_TRUE : JAVA_FALSE; +#endif +#if defined(CN1_GC_INSTRUMENT) + if(ok) { + atomic_fetch_add_explicit(&cn1BibopPagesReleased, 1, memory_order_relaxed); + } +#endif + return ok; +#else + (void)p; + return JAVA_FALSE; +#endif +} + +// Take a released page back into service. Returns whether the page is safe to +// allocate into. +// +// On Darwin a page released with MADV_FREE_REUSABLE is still classified by the +// kernel as reusable storage. MADV_FREE_REUSE is what takes it back out of that +// state and restores the footprint accounting, so if it FAILS the page must not +// be handed to the allocator: the kernel would be free to treat storage that is +// about to hold live objects as discardable, and the process would under-report +// memory it is genuinely using. A page released with an advice that has no +// pairing (MADV_FREE, or Linux MADV_DONTNEED) has nothing to restore and is +// always safe -- which is exactly why the advice kind is recorded per page, so +// an expected rejection there is not mistaken for a real failure here. +static JAVA_BOOLEAN cn1BibopReusePageMemory(CN1BibopPage* p) { +#if !defined(CN1_BIBOP_NO_PAGE_RELEASE) && !defined(_WIN32) + size_t off = cn1BibopReleaseOffset(); + if(off == 0) { + p->gcPageReleased = JAVA_FALSE; + return JAVA_TRUE; + } +#if defined(__APPLE__) + if(!p->gcPageReusableAdvice) { + // Released through the fallback: nothing to restore, but it is leaving + // bibopReleasedPool, so it is no longer waiting for an upgrade. + atomic_fetch_sub_explicit(&bibopFallbackPageCount, 1, memory_order_relaxed); + } + if(p->gcPageReusableAdvice) { + if(cn1ReuseFailInjectOn()) { + return JAVA_FALSE; // fault injection; see the helper + } + if(madvise((char*)p + off, (size_t)CN1_BIBOP_PAGE_SIZE - off, + MADV_FREE_REUSE) != 0) { + cn1PageReuseFailErrno = errno; + return JAVA_FALSE; // leave it released; caller retries later + } + p->gcPageReusableAdvice = JAVA_FALSE; + } +#endif +#if defined(CN1_GC_INSTRUMENT) + atomic_fetch_add_explicit(&cn1BibopPagesReacquired, 1, memory_order_relaxed); +#endif +#endif + p->gcPageReleased = JAVA_FALSE; + return JAVA_TRUE; +} + +// Retry MADV_FREE_REUSABLE on pages that only got the MADV_FREE fallback. +// +// Without this a transient rejection is permanent in effect: the page is filed +// in bibopReleasedPool with gcPageReleased set, so every later trim skips it, +// and it is only ever offered the reusable advice again if the workload happens +// to exhaust the warm pool and reacquire it. A post-burst app that never does +// leaves the page charged to phys_footprint indefinitely -- which is the exact +// figure this feature exists to reduce. +// +// Bounded per trim, and it holds bibopMutex across the syscalls. Both are +// deliberate: the list must not change under the walk, and this path only has +// work to do when a rejection actually happened, which does not occur in normal +// operation at all. +// Matches CN1_BIBOP_RELEASE_PER_SWEEP: an upgrade costs exactly the same single +// madvise a release does, so budgeting it lower only means a burst of transient +// rejections takes proportionally longer to stop being charged. Measured with +// 2000 injected rejections: at 64 per trim the probe ended at 179,776KB with the +// backlog still draining, at this budget it lands on the uninjected figure. +#ifndef CN1_BIBOP_UPGRADE_PER_SWEEP +#define CN1_BIBOP_UPGRADE_PER_SWEEP CN1_BIBOP_RELEASE_PER_SWEEP +#endif +static int cn1BibopUpgradeFallbackPages(void) { +#if !defined(CN1_BIBOP_NO_PAGE_RELEASE) && !defined(_WIN32) && defined(__APPLE__) + size_t off = cn1BibopReleaseOffset(); + if(off == 0) { + return 0; + } + if(atomic_load_explicit(&bibopFallbackPageCount, memory_order_relaxed) <= 0) { + return 0; // nothing is waiting; skip the walk + } + int upgraded = 0; + int attempted = 0; + pthread_mutex_lock(&bibopMutex); + // The budget counts ATTEMPTS, not pages looked at. Counting skips would mean + // that once the leading budget-sized window is upgraded, every later pass + // spends its whole budget re-walking that window and never reaches the + // fallback pages behind it -- they would stay charged forever unless + // allocation happened to reacquire them. Walking past an already-upgraded + // page is a pointer dereference; only the madvise is worth budgeting. + for(CN1BibopPage* p = bibopReleasedPool ; + p != 0 && attempted < CN1_BIBOP_UPGRADE_PER_SWEEP ; + p = p->nextPool) { + if(p->gcPageReusableAdvice) { + continue; // already off the footprint + } + attempted++; + if(!cn1ReusableFailInjectTake() + && madvise((char*)p + off, (size_t)CN1_BIBOP_PAGE_SIZE - off, + MADV_FREE_REUSABLE) == 0) { + p->gcPageReusableAdvice = JAVA_TRUE; + upgraded++; + atomic_fetch_sub_explicit(&bibopFallbackPageCount, 1, memory_order_relaxed); + } + } + pthread_mutex_unlock(&bibopMutex); + return upgraded; +#else + return 0; +#endif +} + +// Release the surplus of bibopFreePool. Called at the end of a sweep, on the GC +// thread. The surplus is UNLINKED under the mutex before any madvise runs, so an +// allocator can never acquire a page while its slot region is being dropped; the +// pages are then published onto bibopReleasedPool in one O(1) splice. +static void cn1BibopTrimFreePool(void) { +#if !defined(CN1_BIBOP_NO_PAGE_RELEASE) && !defined(_WIN32) + if(cn1BibopReleaseOffset() == 0) { + return; + } + pthread_mutex_lock(&bibopMutex); + CN1BibopPage* keepTail = 0; + CN1BibopPage* p = bibopFreePool; + int kept = 0; + while(p != 0 && kept < CN1_BIBOP_FREE_POOL_KEEP) { + keepTail = p; + p = p->nextPool; + kept++; + } + // p is the head of the surplus; bound how much of it this sweep takes. + CN1BibopPage* surplus = p; + CN1BibopPage* surplusTail = 0; + int taken = 0; + while(p != 0 && taken < CN1_BIBOP_RELEASE_PER_SWEEP) { + surplusTail = p; + p = p->nextPool; + taken++; + } + if(surplusTail != 0) { + surplusTail->nextPool = 0; // detach the taken run + if(keepTail != 0) { + keepTail->nextPool = p; // splice any untaken remainder back + } else { + bibopFreePool = p; + } + } + pthread_mutex_unlock(&bibopMutex); + + if(surplusTail == 0) { + // Still worth a pass: pages released through the fallback on an earlier + // trim are charged to the footprint until the reusable advice takes. + int upgradedOnly = cn1BibopUpgradeFallbackPages(); + if(upgradedOnly != 0 && cn1PageReleaseTraceOn()) { + fprintf(stderr, "[PAGE-RELEASE] kept=%d surplus=0 upgraded=%d\n", + kept, upgradedOnly); + } + return; // nothing above the warm cache + } + // Partition the detached run: pages whose memory the kernel actually took go + // to bibopReleasedPool, pages it rejected go back to bibopFreePool. The + // rejected ones are still perfectly good empty pages -- they simply have not + // been handed back yet -- so returning them to the free pool both keeps them + // allocatable and lets a later sweep retry the release. + CN1BibopPage* relHead = 0; + CN1BibopPage* relTail = 0; + CN1BibopPage* retryHead = 0; + CN1BibopPage* retryTail = 0; + int releasedNow = 0; + int rejected = 0; + CN1BibopPage* q = surplus; + while(q != 0) { + CN1BibopPage* next = q->nextPool; + JAVA_BOOLEAN released = q->gcPageReleased; + if(!released) { + released = cn1BibopReleasePageMemory(q); + if(released) { + q->gcPageReleased = JAVA_TRUE; + releasedNow++; + } else { + rejected++; + } + } + if(released) { + q->nextPool = relHead; + relHead = q; + if(relTail == 0) relTail = q; + } else { + q->nextPool = retryHead; + retryHead = q; + if(retryTail == 0) retryTail = q; + } + q = next; + } + int upgraded = cn1BibopUpgradeFallbackPages(); + if(cn1PageReleaseTraceOn()) { + fprintf(stderr, "[PAGE-RELEASE] kept=%d taken=%d released=%d rejected=%d " + "upgraded=%d headerBytes=%zu releaseErrno=%d reuseFailErrno=%d\n", + kept, taken, releasedNow, rejected, upgraded, + cn1BibopReleaseOffset(), cn1PageReleaseReusableErrno, + cn1PageReuseFailErrno); + } + pthread_mutex_lock(&bibopMutex); + if(relTail != 0) { + relTail->nextPool = bibopReleasedPool; + bibopReleasedPool = relHead; + } + if(retryTail != 0) { + retryTail->nextPool = bibopFreePool; + bibopFreePool = retryHead; + } + pthread_mutex_unlock(&bibopMutex); +#endif +} + static CN1BibopPage* cn1BibopNewPage(int ci) { void* mem = cn1BibopRawPage(); if(mem == 0) { return 0; } CN1BibopPage* p = (CN1BibopPage*)mem; + // cn1BibopRawPage hands back indeterminate memory (an arena carved from + // posix_memalign, which malloc may have recycled from its own free list), so + // every header field is garbage until written. cn1BibopFormatPage sets the + // ones it knows about; zeroing first means a field added later cannot be + // silently read before its first assignment, which is the defect this guards + // against. Once per NEW page only -- the pool hit path never reaches here. + memset(p, 0, sizeof(CN1BibopPage)); cn1BibopFormatPage(p, ci); // Publish into the append-only registry: set nextAll (release) BEFORE the // head CAS so a concurrent rescan that reads the new head (acquire) sees a @@ -2664,6 +3175,44 @@ static void cn1BibopMaybeGc(CODENAME_ONE_THREAD_STATE) { np = bibopFreePool; bibopFreePool = np->nextPool; cn1BibopFormatPage(np, ci); + } else if(bibopReleasedPool != 0 || bibopReuseFailedPool != 0) { + // Warm pages are gone; take one whose slot region was handed back to the + // OS. The REUSE call must precede the format, which writes into that + // region (and under CN1_GC_VERIFY writes every slot). + // + // A page whose restore FAILS must not go back at the head: every later + // acquisition would pop the same page, fail again, and allocate a fresh + // arena page, so a single unrestorable page would hide an entire stocked + // pool behind it and the heap would grow without bound. Failures are + // parked on a separate list that is only consulted once the good pool is + // empty, which both keeps them out of the way and still retries them + // eventually if the cause was transient. + for(int attempt = 0 ; np == 0 && attempt < CN1_BIBOP_REUSE_ATTEMPTS ; attempt++) { + CN1BibopPage* cand; + if(bibopReleasedPool != 0) { + cand = bibopReleasedPool; + bibopReleasedPool = cand->nextPool; + } else if(attempt == 0 && bibopReuseFailedPool != 0) { + // Only ever one previously-failed page per acquisition, so a + // permanently unrestorable page costs one syscall and never + // starves the fresh-page fallback. Taken from the HEAD and, on + // failure, returned to the TAIL, so successive acquisitions + // rotate through the parked pages instead of retrying one. + cand = bibopReuseFailedPool; + bibopReuseFailedPool = cand->nextPool; + if(bibopReuseFailedPool == 0) { + bibopReuseFailedTail = 0; + } + } else { + break; + } + if(cn1BibopReusePageMemory(cand)) { + np = cand; + cn1BibopFormatPage(np, ci); + } else { + cn1BibopParkReuseFailure(cand); + } + } } pthread_mutex_unlock(&bibopMutex); if(np == 0) { @@ -3150,6 +3699,70 @@ static void cn1BibopSweep(CODENAME_ONE_THREAD_STATE) { const int cn1GcFaultEarlyFree = 0; #endif CN1BibopPage* list = atomic_exchange_explicit(&bibopSweepStack, (CN1BibopPage*)0, memory_order_acquire); +#if !defined(CN1_BIBOP_NO_PAGE_RELEASE) + // MAJOR SWEEP (issue 5537). The ordinary sweep only ever sees RETIRED pages + // -- ones a thread filled and handed back. A page that was swept while it + // still held live objects goes to bibopPartialPool and is never looked at + // again until some later allocation happens to re-acquire it. So when a big + // live set dies during a quiet period, its pages keep every dead slot: they + // are never re-swept, never become empty, never reach bibopFreePool, and the + // trim below has nothing to hand back. That is why the free pool measured + // empty on the very workload this was meant to fix. + // + // Splicing the partial pools onto the sweep list re-examines them. It is + // correct at any time -- marking reaches an object through its header + // wherever the object lives, so live slots on a partial page carry the + // current epoch exactly as retired pages do, and the full walk rebuilds + // freeList/freeCount from scratch, so re-walking a page is idempotent. + // + // It is NOT free, though: it makes the sweep O(all pages) instead of + // O(retired pages), which is the cost issue 5425 was about. So it runs only + // on a cadence, or immediately when the OS has told us memory is short -- + // the moment actually worth paying for. + if(cn1BibopReleaseOffset() != 0) { + // Run a major sweep when the OS says memory is short, when the app has + // gone QUIET (a burst just ended -- the case that matters, and the case + // where the extra walk costs least), or as a periodic backstop for an app + // that never goes quiet. A cycle driven by allocation volume is none of + // those and keeps the O(retired pages) fast path. + bibopCyclesSinceMajorSweep++; + // "Quiet" has to mean quiet on BOTH allocation paths. A cycle driven by + // legacy volume allocates nothing through BiBOP, so testing + // bibopCycleAllocatedBytes alone would call it quiet and splice every + // partial page in every sweep -- the O(all pages) regression issue 5425 + // fixed, reintroduced for exactly the workload that reported it. + JAVA_BOOLEAN quiet = + ((long long)bibopCycleAllocatedBytes + legacyCycleAllocatedBytes) + < (long long)CN1_BIBOP_MAJOR_SWEEP_QUIET_BYTES; + int major = atomic_load_explicit(&lowMemoryMode, memory_order_relaxed) + || quiet + || bibopCyclesSinceMajorSweep >= CN1_BIBOP_MAJOR_SWEEP_CYCLES; + if(major) { + bibopCyclesSinceMajorSweep = 0; + int spliced = 0; + pthread_mutex_lock(&bibopMutex); + for(int ci = 0 ; ci < CN1_BIBOP_NUM_CLASSES ; ci++) { + CN1BibopPage* p = bibopPartialPool[ci]; + while(p != 0) { + CN1BibopPage* next = p->nextPool; + p->gcMajorSpliced = JAVA_TRUE; + p->nextPool = list; + list = p; + p = next; + spliced++; + } + bibopPartialPool[ci] = 0; + } + pthread_mutex_unlock(&bibopMutex); + if(cn1PageReleaseTraceOn()) { + fprintf(stderr, "[MAJOR-SWEEP] cycle=%d spliced=%d bibopKb=%ld legacyKb=%ld\n", + currentGcMarkValue, spliced, + (long)(bibopCycleAllocatedBytes / 1024), + (long)(legacyCycleAllocatedBytes / 1024)); + } + } + } +#endif int V = currentGcMarkValue; // stable during the sweep (mark done, not yet incremented) long occupiedBytes = 0; long liveBytes = 0; @@ -3167,6 +3780,16 @@ static void cn1BibopSweep(CODENAME_ONE_THREAD_STATE) { while(list != 0) { CN1BibopPage* page = list; list = page->nextPool; + // A page the major sweep pulled out of a PARTIAL pool is a one-off deep + // sample: mostly-dead slots that the ordinary sweep would never have + // looked at again. Feeding it to cn1BibopAdaptAfterSweep drags the + // measured survival ratio down, which halves bibopGcTriggerBytes toward + // its base and buys more collection cycles for no reason -- measured 5 + // cycles to 8 on the issue-5425 workload, eating most of that guard's + // headroom. The page is still swept and still reclaimed; only its + // contribution to the trigger POLICY is withheld. + JAVA_BOOLEAN statsExcluded = page->gcMajorSpliced; + page->gcMajorSpliced = JAVA_FALSE; #ifdef CN1_BIBOP_VALIDATE // INVARIANT: only RETIRED (non-owned) pages reach the sweep. If an OWNED // page (some thread's live bibopCurrent[ci]) is on the sweep stack, the @@ -3232,8 +3855,10 @@ static void cn1BibopSweep(CODENAME_ONE_THREAD_STATE) { page->nextPool = bibopPartialPool[page->classIndex]; bibopPartialPool[page->classIndex] = page; pthread_mutex_unlock(&bibopMutex); - occupiedBytes += (long)n * page->slotSize; - classSlots[page->classIndex] += n; + if(!statsExcluded) { + occupiedBytes += (long)n * page->slotSize; + classSlots[page->classIndex] += n; + } continue; } else if(!page->gcHasMonitors) { // AGED PAST GRACE (even the youngest survivor at gcGraceEpoch < V-1 is @@ -3294,9 +3919,11 @@ static void cn1BibopSweep(CODENAME_ONE_THREAD_STATE) { page->nextPool = bibopFreePool; bibopFreePool = page; pthread_mutex_unlock(&bibopMutex); - occupiedBytes += (long)n * page->slotSize; - reclaimedBytes += (long)n * page->slotSize; - classSlots[page->classIndex] += n; + if(!statsExcluded) { + occupiedBytes += (long)n * page->slotSize; + reclaimedBytes += (long)n * page->slotSize; + classSlots[page->classIndex] += n; + } continue; } // else: all-dead but a BiBOP monitor is live -> fall through to the full walk @@ -3371,11 +3998,13 @@ static void cn1BibopSweep(CODENAME_ONE_THREAD_STATE) { page->freeList = fl; page->freeCount = freeCount; int sampledSlots = n - oldFreeCount; - occupiedBytes += (long)sampledSlots * page->slotSize; - liveBytes += (long)policyLiveCount * page->slotSize; - reclaimedBytes += (long)(sampledSlots - liveCount) * page->slotSize; - classSlots[page->classIndex] += sampledSlots; - classLive[page->classIndex] += policyLiveCount; + if(!statsExcluded) { + occupiedBytes += (long)sampledSlots * page->slotSize; + liveBytes += (long)policyLiveCount * page->slotSize; + reclaimedBytes += (long)(sampledSlots - liveCount) * page->slotSize; + classSlots[page->classIndex] += sampledSlots; + classLive[page->classIndex] += policyLiveCount; + } #ifndef CN1_BIBOP_NO_FASTSWEEP // The monitor (CN1ThreadData) no longer lives in the object header, so the // per-slot "has a monitor" test is gone. Conservatively flag any page that still @@ -3403,6 +4032,10 @@ static void cn1BibopSweep(CODENAME_ONE_THREAD_STATE) { } cn1BibopAdaptAfterSweep(occupiedBytes, liveBytes, reclaimedBytes, classSlots, classLive); + // Hand surplus empty pages back to the OS (issue 5537). Last, so it sees the + // pool this sweep just refilled, and outside the per-page loop so the madvise + // work is batched rather than interleaved with the walk. + cn1BibopTrimFreePool(); } #ifdef CN1_GRACE_AUDIT diff --git a/vm/ByteCodeTranslator/src/nativeMethods.m b/vm/ByteCodeTranslator/src/nativeMethods.m index 4ae19c3ef0d..51e3f05746e 100644 --- a/vm/ByteCodeTranslator/src/nativeMethods.m +++ b/vm/ByteCodeTranslator/src/nativeMethods.m @@ -81,6 +81,13 @@ #if defined(__APPLE__) && defined(__OBJC__) #import +#import +#elif defined(__APPLE__) +// Plain-C Apple target (the clean target emits .c): the same mach and sysctl +// interfaces are available without Objective-C, and back Runtime.freeMemory(). +#include +#include +#include #endif extern _Atomic JAVA_BOOLEAN lowMemoryMode; @@ -2439,9 +2446,74 @@ void releaseForReturnInException(CODENAME_ONE_THREAD_STATE, int cn1LocalsBeginIn threadStateData->callStackOffset--; } +// Bytes this process is metered at. Both Apple branches report phys_footprint +// rather than resident_size, because that is the figure the kernel actually +// charges an app against (it is what jetsam kills on, and what Xcode's memory +// gauge shows). The distinction became load-bearing with the issue-5537 page +// release: memory handed back with MADV_FREE_REUSABLE leaves phys_footprint +// immediately but stays in resident_size until the system is under pressure, so +// a resident_size reading reports a process that has genuinely released memory +// as though it had not -- and an app calling Runtime.freeMemory() to decide +// whether it can afford a cache would never see the memory it just got back. +#if defined(__APPLE__) +static uint64_t cn1PhysFootprint(void) { + task_vm_info_data_t info; + mach_msg_type_number_t count = TASK_VM_INFO_COUNT; + if(task_info(mach_task_self(), TASK_VM_INFO, (task_info_t)&info, &count) == KERN_SUCCESS) { + return (uint64_t)info.phys_footprint; + } + return 0; +} +#endif + +// Resident set of this process in bytes, read from /proc/self/statm (field 2 is +// resident pages). Linux has no phys_footprint; RSS is the right metric there +// because MADV_DONTNEED drops the pages immediately rather than deferring to +// memory pressure, so a release shows up in RSS the moment it happens. +#if defined(__linux__) +static uint64_t cn1LinuxResidentBytes(void) { + FILE* f = fopen("/proc/self/statm", "r"); + if(f == 0) { + return 0; + } + unsigned long total = 0, resident = 0; + int n = fscanf(f, "%lu %lu", &total, &resident); + fclose(f); + if(n != 2) { + return 0; + } + long ps = sysconf(_SC_PAGESIZE); + if(ps <= 0) { + return 0; + } + return (uint64_t)resident * (uint64_t)ps; +} +#endif + JAVA_LONG java_lang_Runtime_totalMemoryImpl___R_long(CODENAME_ONE_THREAD_STATE) { #if defined(__APPLE__) && defined(__OBJC__) return [NSProcessInfo processInfo].physicalMemory; +#elif defined(__linux__) + { + long pages = sysconf(_SC_PHYS_PAGES); + long ps = sysconf(_SC_PAGESIZE); + if(pages > 0 && ps > 0) { + return (JAVA_LONG)((uint64_t)pages * (uint64_t)ps); + } + } + return 1024*1024*1024; +#elif defined(__APPLE__) + // Plain-C Apple target (the translator's clean target emits .c, so the + // __OBJC__ branch above is unavailable there). sysctl reports the same + // figure NSProcessInfo.physicalMemory does. + { + uint64_t total = 0; + size_t len = sizeof(total); + if(sysctlbyname("hw.memsize", &total, &len, NULL, 0) == 0 && total > 0) { + return (JAVA_LONG)total; + } + } + return 1024*1024*1024; #else // TODO: implement for other platforms return 1024*1024*1024; @@ -2449,14 +2521,18 @@ JAVA_LONG java_lang_Runtime_totalMemoryImpl___R_long(CODENAME_ONE_THREAD_STATE) } JAVA_LONG java_lang_Runtime_freeMemoryImpl___R_long(CODENAME_ONE_THREAD_STATE) { -#if defined(__APPLE__) && defined(__OBJC__) - struct task_basic_info info; - mach_msg_type_number_t size = sizeof(info); - kern_return_t kerr = task_info(mach_task_self(), - TASK_BASIC_INFO, - (task_info_t)&info, - &size); - return [NSProcessInfo processInfo].physicalMemory - info.resident_size; +#if defined(__APPLE__) + { + JAVA_LONG total = java_lang_Runtime_totalMemoryImpl___R_long(threadStateData); + uint64_t used = cn1PhysFootprint(); + return used == 0 ? total : total - (JAVA_LONG)used; + } +#elif defined(__linux__) + { + JAVA_LONG total = java_lang_Runtime_totalMemoryImpl___R_long(threadStateData); + uint64_t used = cn1LinuxResidentBytes(); + return used == 0 ? total : total - (JAVA_LONG)used; + } #else // TODO: implement for other platforms return 1024*1024*1024; diff --git a/vm/benchmarks/src/com/bench/MajorSweepMix.java b/vm/benchmarks/src/com/bench/MajorSweepMix.java new file mode 100644 index 00000000000..1c1b1c33482 --- /dev/null +++ b/vm/benchmarks/src/com/bench/MajorSweepMix.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ + +package com.bench; + +/** + * The workload that exposes a major-sweep misclassification: a large BiBOP + * survivor set driven by heavy LEGACY churn. + * + *

Both halves matter. The survivors are small (BiBOP-resident) objects that + * stay live for the whole run, so their pages sit in bibopPartialPool where the + * ordinary sweep never revisits them -- that is the population a major sweep has + * to walk. The churn is large arrays, which take the legacy calloc path, so it + * is what actually drives the collection cycles while contributing nothing to + * bibopBytesSinceGc. + * + *

A quiet-cycle test that looks only at BiBOP volume therefore calls every + * one of these cycles quiet and splices every partial page into every sweep, + * which is the O(all pages) cost issue 5425 removed. Measured here with + * CN1_LOG_PAGE_RELEASE=1 and CN1_GC_LOG_CYCLES=1, counting [MAJOR-SWEEP] lines + * against [GC-CYCLE] lines: 36 major sweeps in 38 cycles when the quiet test + * ignores legacy bytes, 2 in 39 when it includes them. + * + *

The churn phase runs for a fixed wall duration rather than a fixed count, + * so the number of 200ms-paced collection cycles is machine-independent. + */ +public class MajorSweepMix { + /** Retained for the whole run, so these pages stay in the partial pool. */ + static Object[] survivors; + + /** Small enough for the BiBOP page heap (at or under CN1_BIBOP_MAX_OBJECT). */ + private static final int SMALL_BYTES = 256; + + /** Comfortably above CN1_BIBOP_MAX_OBJECT, so the churn is legacy-path. */ + private static final int LARGE_BYTES = 65536; + + private static final int SURVIVOR_COUNT = 400000; + private static final long CHURN_MS = 12000; + + public static void main(String[] args) { + Object[] live = new Object[SURVIVOR_COUNT]; + for (int i = 0; i < SURVIVOR_COUNT; i++) { + byte[] o = new byte[SMALL_BYTES]; + o[0] = (byte) i; + live[i] = o; + } + survivors = live; + + long checksum = 0; + long deadline = System.currentTimeMillis() + CHURN_MS; + int i = 0; + while (System.currentTimeMillis() < deadline) { + // Check the clock once per batch; the call would otherwise dominate. + for (int b = 0; b < 64; b++, i++) { + byte[] big = new byte[LARGE_BYTES]; + // Touch every page: an untouched calloc'd block costs no memory. + for (int off = 0; off < LARGE_BYTES; off += 4096) { + big[off] = (byte) (i + off); + } + checksum += big[0] + big[LARGE_BYTES - 1]; + } + } + checksum += ((byte[]) survivors[0])[0]; + System.out.println("ALLOCATIONS=" + i); + System.out.println("RESULT=" + checksum); + } +} diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/BibopPageFloorIntegrationTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/BibopPageFloorIntegrationTest.java new file mode 100644 index 00000000000..ba29bd68308 --- /dev/null +++ b/vm/tests/src/test/java/com/codename1/tools/translator/BibopPageFloorIntegrationTest.java @@ -0,0 +1,487 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * Guards the issue-5537 fix: BiBOP now hands surplus empty pages back to the OS. + * + *

Before the fix a swept-empty page went to bibopFreePool and stayed resident + * for the life of the process -- there was no munmap, madvise or free anywhere in + * the path. Because pages are ALSO size-class segregated, that memory was not + * merely idle but unusable by anything except a future block of + * CN1_BIBOP_MAX_OBJECT bytes or less, so a past small-object peak permanently + * crowded out later large or native allocations (an image buffer, a Metal + * texture, a glyph atlas). On iOS that is subtracted from a jetsam ceiling of + * roughly 1.4GB.

+ * + *

The app holds 192MB of 256-byte objects, drops them, forces six collection + * cycles, then allocates a 192MB large-buffer "texture" set, then drops those and + * allocates the identical set again. Measured, in phys_footprint KB:

+ * + * 
+ *   phase                    baseKB     heldKB  releasedKB    | before fix
+ *   small-warmup               2240     269536       90640    | 269504 released
+ *   texture-after-small       90640     287744      287744    | 467264 held
+ *   texture-after-texture    287744     287744      287744    | 467264 held
+ *
+ * + *

Two independent things are asserted. First the fix works: the warm-up gives + * back about 66% of its footprint where it previously gave back zero. Second the + * ORIGINAL finding still holds and is still measured -- the texture set costs + * full price over BiBOP-freed memory (the treatment) and nothing over + * legacy-freed memory (the control), because a 64KB page belongs to one size + * class and can only ever hand out blocks of 512 bytes or less. That contrast is + * why returning the pages matters at all, and keeping it in the test means a + * regression that silently stopped releasing them is caught by the peak.

+ * + *

Measured as PHYS_FOOTPRINT, read by the app through Runtime, not as resident + * size. That distinction is essential here: MADV_FREE_REUSABLE removes pages from + * phys_footprint immediately but leaves them in resident_size until the system is + * under pressure, so an RSS probe reports the fix as doing nothing. phys_footprint + * is also the figure the kernel actually meters an app against.

+ * + *

Note this needs no race with the collector. Every phase allocates, holds, + * drops, then forces collection and waits, so anything still resident is held by + * design rather than by a pacing accident -- which is why the numbers barely move + * between an idle host and a loaded one, and why these can be real gates.

+ * + *

Tagged {@code benchmark}: it needs a translate-and-build and peaks near + * 300MB.

+ */ +@Tag("benchmark") +class BibopPageFloorIntegrationTest { + + /** Keep in sync with TEXTURE_BYTES * TEXTURE_COUNT in BibopPageFloorApp. */ + private static final long TEXTURE_SET_KB = 12L * 16 * 1024; + + /** + * A phase that drops its live set and forces six collection cycles must give + * most of it back. Before the fix BiBOP returned exactly nothing (269504KB + * held, 269504KB after release); with it the same phase drops to 90640KB, + * about 66% returned. + * + *

The 34% that stays is the page-header tax, and it is a real limit + * rather than slack in the measurement: only whole system pages can be + * released, the CN1BibopPage header lives at the base of its 64KB page, and + * arm64 (device, and the Apple-silicon simulator) has a 16KB system page -- + * so 16KB of every 64KB page has to stay resident. On a 4KB-page target the + * same code returns about 94%. Closing that gap means moving the header out + * of the page, which the address-to-page mask in cn1ConservativeResolve and + * the nextAll registry both depend on; that is a redesign, not a tweak.

+ */ + private static final double FLOOR_MAX_RETAINED_FRACTION = 0.55; + + /** + * What the texture set costs on top of the post-release floor. Before the + * fix this was the full set (196992KB of 196608KB) because pooled pages + * could not serve a large block; that is still true, but the pool is now + * much smaller, so the peak this phase reaches is what actually improved + * (467264KB before, 287744KB after). Kept as a gate because a regression + * that stopped releasing pages would push it back up. + */ + private static final double TREATMENT_MIN_COST_FRACTION = 0.5; + + /** + * The control must pay almost nothing, because legacy-freed memory is + * reusable. Measured 0KB. The budget absorbs allocator bookkeeping. + */ + private static final long CONTROL_MAX_COST_KB = 32 * 1024; + + @Test + void bibopReclaimedPagesAreUnavailableToLargeAllocations() throws Exception { + Parser.cleanup(); + + List tempDirs = new ArrayList<>(); + try { + runFloorProbe(tempDirs); + } finally { + for (Path dir : tempDirs) { + deleteRecursively(dir); + } + } + } + + private void runFloorProbe(List tempDirs) throws Exception { + Path sourceDir = Files.createTempDirectory("bibop-page-floor-sources"); + Path classesDir = Files.createTempDirectory("bibop-page-floor-classes"); + Path javaApiDir = Files.createTempDirectory("bibop-page-floor-javaapi"); + tempDirs.add(sourceDir); + tempDirs.add(classesDir); + tempDirs.add(javaApiDir); + + Path source = sourceDir.resolve("BibopPageFloorApp.java"); + Files.write(source, loadAppSource().getBytes(StandardCharsets.UTF_8)); + + CompilerHelper.CompilerConfig config = selectCompiler(); + if (config == null) { + fail("No compatible compiler available for the BiBOP page-floor probe"); + } + assertTrue(CompilerHelper.isJavaApiCompatible(config), + "JDK " + config.jdkVersion + " must target matching bytecode level for JavaAPI"); + + CompilerHelper.compileJavaAPI(javaApiDir, config); + + List compileArgs = new ArrayList<>(); + compileArgs.add("-source"); + compileArgs.add(config.targetVersion); + compileArgs.add("-target"); + compileArgs.add(config.targetVersion); + if (CompilerHelper.useClasspath(config)) { + compileArgs.add("-classpath"); + compileArgs.add(javaApiDir.toString()); + } else { + compileArgs.add("-bootclasspath"); + compileArgs.add(javaApiDir.toString()); + compileArgs.add("-Xlint:-options"); + } + compileArgs.add("-d"); + compileArgs.add(classesDir.toString()); + compileArgs.add(source.toString()); + + int compileResult = CompilerHelper.compile(config.jdkHome, compileArgs); + assertEquals(0, compileResult, + "BibopPageFloorApp should compile. " + CompilerHelper.getLastErrorLog()); + + String javaOutput = runJavaMain(config, classesDir, javaApiDir); + String javaResult = extractLine(javaOutput, "RESULT="); + assertTrue(javaResult.startsWith("RESULT="), + "JavaSE should produce RESULT=. Output: " + javaOutput); + + CompilerHelper.copyDirectory(javaApiDir, classesDir); + + Path outputDir = Files.createTempDirectory("bibop-page-floor-output"); + tempDirs.add(outputDir); + CleanTargetIntegrationTest.runTranslator(classesDir, outputDir, "BibopPageFloorApp"); + + Path distDir = outputDir.resolve("dist"); + Path cmakeLists = distDir.resolve("CMakeLists.txt"); + assertTrue(Files.exists(cmakeLists), "Translator should emit a CMake project"); + CleanTargetIntegrationTest.replaceLibraryWithExecutableTarget(cmakeLists, "BibopPageFloorApp-src"); + + Path buildDir = distDir.resolve("build"); + Files.createDirectories(buildDir); + List cmakeArgs = new ArrayList<>(Arrays.asList( + "cmake", + "-S", distDir.toString(), + "-B", buildDir.toString(), + "-DCMAKE_BUILD_TYPE=Release", + "-DCMAKE_C_COMPILER=clang", + "-DCMAKE_OBJC_COMPILER=clang")); + if (isAppleSilicon()) { + // Build for the architecture the product actually ships on. This + // repo's JDK 8 is an x64 build, so on Apple silicon Maven runs under + // Rosetta and everything it spawns -- cmake, clang -- defaults to + // x86_64. That matters here and nowhere else: under Rosetta + // MADV_FREE_REUSABLE returns success but phys_footprint never drops, + // so a translated x86_64 binary reports the page release as doing + // nothing while the identical arm64 binary returns 66% of its + // footprint. Measured, same source, same 192MB warm-up: + // arm64 269552KB -> 90656KB, x86_64 263813KB -> 263821KB. + cmakeArgs.add("-DCMAKE_OSX_ARCHITECTURES=arm64"); + } + CleanTargetIntegrationTest.runCommand(cmakeArgs, distDir); + CleanTargetIntegrationTest.runCommand(Arrays.asList("cmake", "--build", buildDir.toString()), distDir); + + Path executable = buildDir.resolve("BibopPageFloorApp"); + assertTrue(Files.exists(executable), "ParparVM build should produce a runnable executable"); + + String vmOutput = runVm(executable, buildDir); + + assertTrue(vmOutput.contains("BIBOP_PAGE_FLOOR_DONE"), + "The probe must run to completion. Output: " + vmOutput); + assertEquals(javaResult, extractLine(vmOutput, "RESULT="), + "Both runtimes must compute the same answer, so a footprint figure cannot come " + + "from quietly allocating less\n--- JavaSE ---\n" + javaOutput + + "\n--- ParparVM ---\n" + vmOutput); + + Map> marks = parseMarks(vmOutput); + StringBuilder report = new StringBuilder(); + report.append(String.format("%-24s %10s %10s %11s %11s%n", + "phase", "baseKB", "heldKB", "releasedKB", "minAfterKB")); + for (Map.Entry> e : marks.entrySet()) { + Map m = e.getValue(); + report.append(String.format("%-24s %10d %10d %11d %11d%n", e.getKey(), + m.containsKey("BASELINE") ? m.get("BASELINE") : -1, + m.containsKey("HELD") ? m.get("HELD") : -1, + m.containsKey("RELEASED") ? m.get("RELEASED") : -1, + m.containsKey("MINAFTER") ? m.get("MINAFTER") : -1)); + } + System.err.println("[BibopPageFloorIntegrationTest] texture set " + TEXTURE_SET_KB + + "KB, phys_footprint\n" + report); + + long warmupHeld = require(marks, "small-warmup", "HELD", report); + long warmupReleased = require(marks, "small-warmup", "MINAFTER", report); + long treatmentBase = require(marks, "texture-after-small", "BASELINE", report); + long treatmentHeld = require(marks, "texture-after-small", "HELD", report); + long controlBase = require(marks, "texture-after-texture", "BASELINE", report); + long controlHeld = require(marks, "texture-after-texture", "HELD", report); + + // A target whose Runtime memory natives are still stubs reports 0 for every + // phase; so does a run where task_info was unavailable. Nothing can be + // measured then, and failing would report a porting/environment gap as a + // memory regression. + org.junit.jupiter.api.Assumptions.assumeTrue(warmupHeld > 0, + "This run could not read phys_footprint through Runtime, so the probe cannot be " + + "measured here.\n" + report); + + assertTrue(warmupHeld > TEXTURE_SET_KB, + "small-warmup only reached " + warmupHeld + "KB holding a " + TEXTURE_SET_KB + + "KB live set, so the probe never built a pool worth releasing and the " + + "rest of this test proves nothing.\n" + report); + + // 1. THE FIX: surplus empty pages are handed back to the OS. + long floorBudget = (long) (warmupHeld * FLOOR_MAX_RETAINED_FRACTION); + assertTrue(warmupReleased <= floorBudget, + "small-warmup dropped its entire live set, yet the LOWEST phys_footprint seen at " + + "any point over the rest of the run was " + warmupReleased + "KB against " + + warmupHeld + "KB held (budget " + floorBudget + "KB). This is a minimum " + + "over the whole remainder, not a reading at one instant, so it cannot be " + + "a matter of the collector being slow on a loaded runner -- the pages " + + "were never given back. Check cn1BibopTrimFreePool, the major sweep that " + + "refills bibopFreePool from the partial pools, and that " + + "CN1_BIBOP_NO_PAGE_RELEASE is not set.\n" + report); + + // 2. The original finding, still measured: pooled pages cannot serve a + // large block, so the texture set costs full price over them... + long treatmentCost = treatmentHeld - treatmentBase; + long treatmentFloor = (long) (TEXTURE_SET_KB * TREATMENT_MIN_COST_FRACTION); + assertTrue(treatmentCost >= treatmentFloor, + "texture-after-small cost only " + treatmentCost + "KB for a " + TEXTURE_SET_KB + + "KB texture set (expected at least " + treatmentFloor + "KB), which would " + + "mean the BiBOP pool DID serve a large allocation. That contradicts the " + + "size-class design, so check CN1_BIBOP_MAX_OBJECT and the page pooling " + + "before trusting it.\n" + report); + + // 3. ...while the identical allocation over a legacy-freed hole is free. + // Without this control the treatment proves nothing: it could simply + // be that large allocations always cost full price here. + long controlCost = controlHeld - controlBase; + if (controlCost > CONTROL_MAX_COST_KB) { + // Reported, not gated. The control assumes the hole the first texture + // set left behind is still held by malloc, which is true when this + // test runs alone but not when the machine is under memory pressure + // from parallel surefire forks -- malloc returns large blocks to the + // OS and the second set has to fault them back in, so it pays full + // price for a reason that has nothing to do with BiBOP. Measured 0KB + // alone, 196720KB under a full-suite run. + System.err.println("[BibopPageFloorIntegrationTest] control inconclusive this run: " + + "texture-after-texture cost " + controlCost + "KB, so malloc did not retain " + + "the hole from the previous phase and the treatment/control contrast cannot " + + "be read. The page-release assertions above are unaffected."); + } + + // 4. The consequence that matters on device: the peak the treatment + // reaches. Unfixed, the released-but-retained pool stacked under the + // texture set and the peak was warmupHeld + the full texture set. + long unfixedPeak = warmupHeld + TEXTURE_SET_KB; + long peakBudget = warmupHeld + (long) (TEXTURE_SET_KB * 0.9); + assertTrue(treatmentHeld < peakBudget, + "texture-after-small peaked at " + treatmentHeld + "KB. Without page release the " + + "peak is the whole small-object pool plus the whole texture set (about " + + unfixedPeak + "KB); the budget here is " + peakBudget + "KB. A peak this " + + "high means the pool was still resident underneath the textures, which " + + "is the issue-5537 shape.\n" + report); + + System.err.println("[BibopPageFloorIntegrationTest] page release returned " + + (warmupHeld - warmupReleased) + "KB of " + warmupHeld + "KB (" + + (100 - (warmupReleased * 100 / warmupHeld)) + "%, measured as the minimum over " + + "the rest of the run); texture peak " + treatmentHeld + "KB against " + + unfixedPeak + "KB unfixed."); + } + + private long require(Map> marks, String phase, String marker, + StringBuilder report) { + Map m = marks.get(phase); + if (m == null || !m.containsKey(marker)) { + fail("Missing " + marker + " footprint for phase " + phase + " in " + marks.keySet() + + "\n" + report); + } + return m.get(marker); + } + + private Map> parseMarks(String output) { + Pattern p = Pattern.compile( + "ARM_(BASELINE|BEGIN|HELD|RELEASED|MINAFTER) name=(\\S+) tMs=\\d+ footprintKb=(\\d+)"); + Map> marks = new LinkedHashMap<>(); + for (String line : output.split("\\R")) { + Matcher m = p.matcher(line); + if (m.find()) { + Map phases = marks.get(m.group(2)); + if (phases == null) { + phases = new LinkedHashMap<>(); + marks.put(m.group(2), phases); + } + phases.put(m.group(1), Long.parseLong(m.group(3))); + } + } + return marks; + } + + /** + * True on Apple silicon, including when this JVM is itself running under + * Rosetta -- hw.optional.arm64 describes the hardware, not the caller. + */ + private static boolean isAppleSilicon() { + if (!System.getProperty("os.name").toLowerCase().contains("mac")) { + return false; + } + try { + ProcessBuilder pb = new ProcessBuilder("sysctl", "-n", "hw.optional.arm64"); + pb.redirectErrorStream(true); + Process p = pb.start(); + String out; + try (BufferedReader r = new BufferedReader( + new InputStreamReader(p.getInputStream(), StandardCharsets.UTF_8))) { + out = r.readLine(); + } + p.waitFor(); + return "1".equals(out == null ? "" : out.trim()); + } catch (Exception e) { + return false; + } + } + + /** Runs the translated binary. Memory is reported by the app, not sampled here. */ + private String runVm(Path executable, Path workingDir) throws Exception { + ProcessBuilder builder = new ProcessBuilder(executable.toAbsolutePath().toString()); + builder.directory(workingDir.toFile()); + // Do NOT merge stderr into stdout. The VM's env-gated tracers write to + // stderr, and a merged write can land in the middle of a marker line -- + // observed as a phase silently missing from the table because its marker + // had a tracer line spliced through it. The footprint columns are the + // evidence here; set CN1_LOG_PAGE_RELEASE=1 by hand when you want the + // per-sweep page counts alongside them. + builder.redirectError(ProcessBuilder.Redirect.INHERIT); + Process process = builder.start(); + String output; + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8))) { + output = reader.lines().collect(Collectors.joining("\n")); + } + assertEquals(0, process.waitFor(), + "ParparVM run should exit cleanly. Output: " + output); + return output; + } + + private String loadAppSource() throws Exception { + java.io.InputStream in = BibopPageFloorIntegrationTest.class + .getResourceAsStream("/com/codename1/tools/translator/BibopPageFloorApp.java"); + assertNotNull(in, "BibopPageFloorApp.java test resource should exist"); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))) { + return reader.lines().collect(Collectors.joining("\n")) + "\n"; + } + } + + private String runJavaMain(CompilerHelper.CompilerConfig config, Path classesDir, Path javaApiDir) + throws Exception { + String javaExe = config.jdkHome.resolve("bin").resolve("java").toString(); + if (System.getProperty("os.name").toLowerCase().contains("win")) { + javaExe += ".exe"; + } + ProcessBuilder pb = new ProcessBuilder( + javaExe, + "-Xmx1g", + "-cp", + classesDir + System.getProperty("path.separator") + javaApiDir, + "BibopPageFloorApp"); + pb.redirectErrorStream(true); + Process process = pb.start(); + String output; + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8))) { + output = reader.lines().collect(Collectors.joining("\n")); + } + assertEquals(0, process.waitFor(), "JVM run should exit cleanly. Output: " + output); + return output; + } + + private String extractLine(String output, String prefix) { + for (String line : output.split("\\R")) { + if (line.startsWith(prefix)) { + return line.trim(); + } + } + return ""; + } + + private CompilerHelper.CompilerConfig selectCompiler() { + String[] preferredTargets = {"11", "17", "21", "25", "1.8"}; + for (String target : preferredTargets) { + List configs = CompilerHelper.getAvailableCompilers(target); + for (CompilerHelper.CompilerConfig config : configs) { + if (CompilerHelper.isJavaApiCompatible(config)) { + return config; + } + } + } + return null; + } + + private static void deleteRecursively(Path root) { + if (root == null || !Files.exists(root)) { + return; + } + final java.io.IOException[] firstFailure = new java.io.IOException[1]; + try (java.util.stream.Stream walk = Files.walk(root)) { + walk.sorted(java.util.Comparator.reverseOrder()).forEach(p -> { + try { + Files.deleteIfExists(p); + } catch (java.io.IOException e) { + if (firstFailure[0] == null) { + firstFailure[0] = e; + } + } + }); + } catch (java.io.IOException e) { + if (firstFailure[0] == null) { + firstFailure[0] = e; + } + } + if (firstFailure[0] != null) { + System.err.println("BibopPageFloorIntegrationTest: temp cleanup incomplete under " + + root + " (first failure: " + firstFailure[0] + ")"); + } + } +} diff --git a/vm/tests/src/test/resources/com/codename1/tools/translator/BibopPageFloorApp.java b/vm/tests/src/test/resources/com/codename1/tools/translator/BibopPageFloorApp.java new file mode 100644 index 00000000000..b18f0f767e6 --- /dev/null +++ b/vm/tests/src/test/resources/com/codename1/tools/translator/BibopPageFloorApp.java @@ -0,0 +1,411 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ + +/** + * Proves that reclaimed BiBOP pages are permanently unavailable to any + * allocation that is not a small object -- the mechanism by which a past + * small-object peak crowds out a later large/native allocation such as a Metal + * texture, an image surface or a glyph atlas. + * + *

The claim is structural, not statistical. BiBOP is a segregated-fits page + * heap: a 64KB page belongs to one size class, and cn1BibopAlloc can hand its + * slots only to blocks at or under CN1_BIBOP_MAX_OBJECT (512). A swept page goes + * to bibopFreePool or bibopPartialPool, and there is no munmap, no madvise and + * no free anywhere in that path -- cn1_globals.m states it outright: "BiBOP + * never free()s a page (swept pages are pooled)". Anything larger, including + * every buffer a native texture would be built from, takes the legacy calloc + * path and cannot touch those pooled pages. + * + *

So the two consequences are testable without racing the collector at all, + * which is deliberate: this app never depends on the mutator outrunning the GC, + * it allocates a live set, drops it, forces collection and waits. Any resident + * memory still held after that is held BY DESIGN, not by a pacing accident. + * + *

PHASE ORDER, and what each phase is for: + * + *

    + *
  1. {@code texture-cold} -- allocate TEXTURE_BYTES of large buffers on a clean + * heap, hold, release. Establishes what a large allocation costs when no + * BiBOP pool exists. This is the control.
    2. + *
  2. {@code small-1}, {@code small-2} -- allocate a large LIVE set of small + * (BiBOP-resident) objects, hold, drop, collect, wait. The pool must grow to + * hold them. After the drop every one of those pages is free, so a heap that + * returned memory would fall back to baseline. The second round is bigger + * than the first, so the floor can be checked for monotonicity.
    3. + *
  3. {@code texture-warm} -- allocate exactly the same large buffers again, now + * with a fully populated BiBOP free pool sitting underneath. If those pooled + * pages were available to a large allocation, this phase would cost nothing. + * If they are not, it costs the same as texture-cold and stacks on top of the + * floor.
    4. + *
+ * + *

The proof is the comparison of texture-warm's cost against texture-cold's, + * and of the post-release floor against baseline. On a device the sum of those + * two -- an unreclaimable small-object floor plus a live texture working set -- + * is what has to fit under the roughly 1.4GB jetsam ceiling. + * + *

Memory is reported as PHYS_FOOTPRINT, read through Runtime, not as resident + * size. That distinction is the whole measurement here: memory handed back with + * MADV_FREE_REUSABLE leaves phys_footprint immediately but stays in + * resident_size until the system is actually under pressure, so an RSS probe + * shows a process that has released memory as though it had not -- and + * phys_footprint is the figure the kernel meters an app against anyway. Phase + * boundaries are also stamped with System.currentTimeMillis() so a harness can + * cross-check against an external sampler. + * + *

NOTE ON FIDELITY. The "texture" buffers here are large byte[] on the legacy + * calloc path, not literal MTLTexture allocations -- the clean target has no + * Metal. For this measurement they are equivalent: both are malloc/mmap requests + * against the same address space that cannot be served from a 512-byte-class + * BiBOP page. What the model does NOT capture is texture memory the driver + * places in a separate device heap; on unified-memory Apple silicon it is the + * same physical budget, which is the case that matters here. + */ +public class BibopPageFloorApp { + + /** + * Small-object payload. Total block size (array header + data + trailing + * slot pointer) must stay at or below CN1_BIBOP_MAX_OBJECT for these to be + * served from the page heap rather than the legacy path. + */ + private static final int SMALL_BYTES = 256; + + /** + * One "texture": a plausible surface-sized buffer. Far above + * CN1_BIBOP_MAX_OBJECT, so it always takes the legacy calloc path. + */ + private static final int TEXTURE_BYTES = 16 * 1024 * 1024; + + /** Total live texture working set per texture phase. */ + private static final int TEXTURE_COUNT = 12; + + /** + * Live small-object bytes in the warm-up phase. Deliberately equal to the + * texture working set, so "the pool is big enough to have served it" is not + * an available explanation for any shortfall. + */ + private static final long SMALL_LIVE_BYTES = (long) TEXTURE_BYTES * TEXTURE_COUNT; + + /** One write per 4096 bytes materializes every page of a calloc'd block. */ + private static final int PAGE_STRIDE = 4096; + + /** + * Settle after dropping a phase's live set. Deliberately generous: the whole + * question is whether memory comes back AT ALL, so the answer must not be + * "the collector had not finished yet". System.gc() is asynchronous (it sets + * forceGc and notifies the collector thread, then returns), so each round is + * a request plus a pause long enough for a full cycle to land. + */ + private static final int SETTLE_MIN_ROUNDS = 4; + private static final int SETTLE_MAX_ROUNDS = 20; + private static final int SETTLE_PLAIN_MIN_ROUNDS = 4; + private static final int SETTLE_PLAIN_MAX_ROUNDS = 12; + private static final long SETTLE_PAUSE_MS = 250; + + /** + * Rounds that must ALL come back stable before a settle is believed. One + * stable round is not enough: reclamation does not begin immediately, so + * early rounds look stable simply because nothing has started coming back + * yet -- measured on Linux, where the first pages were not released until + * after a four-round settle had already concluded and the phase read as + * having released nothing. + */ + private static final int SETTLE_STABLE_STREAK = 3; + + /** Stack frames overwritten before a settle; see scrubStack. */ + private static final int SCRUB_DEPTH = 512; + + /** + * A settle stops once a round frees less than this. Reclamation is not a + * fixed number of cycles: a BiBOP object needs three sweeps to die (fresh + * mark -1 is promoted, and death is mark < V-1), the major sweep that + * refills the free pool runs on a cadence, and how many cycles land in a + * given wall-clock window depends on the machine. A fixed round count + * therefore measures the runner, not the collector -- measured directly: + * six rounds was enough on macOS but not on the Linux CI runner, where the + * footprint was still falling when the reading was taken and the phase + * looked like it had released nothing. + */ + private static final long SETTLE_STABLE_KB = 2048; + + /** + * How long a phase keeps its live set REACHABLE before dropping it. Without + * this the allocation itself takes tens of milliseconds and the harness + * sampler gets no reading at all while the data is held, which is the + * measurement the whole test turns on. The live set is touched between + * pauses so it is provably reachable across the whole window. + */ + private static final int HOLD_ROUNDS = 8; + private static final long HOLD_PAUSE_MS = 200; + + private static long checksum; + + /** + * Lowest footprint seen at any point after the warm-up's live set was + * dropped, and the flag that starts tracking it. + * + *

This is what the release assertion reads, rather than the footprint at + * some chosen instant. Reclamation is asynchronous and its LATENCY is + * load-dependent: measured, the drop lands about 2s after the drop on an + * idle macOS host, about 7s in an idle Linux container, and had still not + * landed 15s in on a CI runner where surefire was running this probe + * alongside a dozen other forks. The claim under test is that the pages come + * back, not that they come back within some number of seconds, so waiting on + * a deadline was testing the runner's load rather than the collector. The + * minimum over the remainder of the run answers the actual question and + * cannot be made to pass by a release that never happens. + */ + private static long minFootprintAfterDrop = Long.MAX_VALUE; + private static boolean trackMinFootprint; + + public static void main(String[] args) { + System.out.println("CONFIG smallBytes=" + SMALL_BYTES + + " textureBytes=" + TEXTURE_BYTES + + " textureCount=" + TEXTURE_COUNT + + " smallLiveBytes=" + SMALL_LIVE_BYTES); + + // 1. Grow the BiBOP page pool to SMALL_LIVE_BYTES, then give every page + // back to it. Nothing large has been allocated yet, so the process's + // malloc heap holds no large-block hole a texture could reuse. + smallPhase("small-warmup", SMALL_LIVE_BYTES); + + // 2. TREATMENT. Allocate the texture set on top of that fully populated + // BiBOP free pool. If pooled pages could serve a large block this + // costs nothing; if they cannot, it costs the full texture set. + texturePhase("texture-after-small"); + + // 3. CONTROL. Drop those textures and allocate the identical set again. + // Now the hole underneath was freed by the LEGACY path rather than by + // BiBOP, so a reuse-capable allocator must absorb it for free. Any + // difference between phase 2 and phase 3 is attributable to which + // allocator released the memory, with the size, the type and the + // access pattern all held identical. + texturePhase("texture-after-texture"); + + System.out.println("ARM_MINAFTER name=small-warmup tMs=" + System.currentTimeMillis() + + " footprintKb=" + + (minFootprintAfterDrop == Long.MAX_VALUE ? 0 : minFootprintAfterDrop)); + System.out.println("RESULT=" + checksum); + System.out.println("BIBOP_PAGE_FLOOR_DONE"); + } + + /** Allocates and holds a large live set of BiBOP-resident objects, then drops it. */ + private static void smallPhase(String name, long liveBytes) { + int count = (int) (liveBytes / SMALL_BYTES); + beginPhase(name); + + byte[][] live = new byte[count][]; + for (int i = 0; i < count; i++) { + byte[] o = new byte[SMALL_BYTES]; + o[0] = (byte) i; + o[SMALL_BYTES - 1] = (byte) (i >> 8); + live[i] = o; + } + long phaseChecksum = 0; + for (int i = 0; i < count; i += 997) { + phaseChecksum += live[i][0] + live[i][SMALL_BYTES - 1]; + } + phaseChecksum += hold(live, SMALL_BYTES); + long heldKb = footprintKb(); + endPhase(name, "objects=" + count + " liveBytes=" + liveBytes); + + live = null; + releasePhase(name, heldKb, true); + // Everything from here on is after the warm-up's live set died, so every + // reading contributes to the minimum the release assertion reads. + trackMinFootprint = true; + checksum = checksum * 131 + phaseChecksum; + } + + /** Allocates and holds a large live set of legacy-path buffers, then drops it. */ + private static void texturePhase(String name) { + beginPhase(name); + + byte[][] textures = new byte[TEXTURE_COUNT][]; + for (int i = 0; i < TEXTURE_COUNT; i++) { + byte[] t = new byte[TEXTURE_BYTES]; + // Touch every page: an untouched calloc'd block costs no resident + // memory, and a texture that is never written is not a texture. + for (int off = 0; off < TEXTURE_BYTES; off += PAGE_STRIDE) { + t[off] = (byte) (i + off); + } + textures[i] = t; + } + long phaseChecksum = 0; + for (int i = 0; i < TEXTURE_COUNT; i++) { + phaseChecksum += textures[i][0] + textures[i][TEXTURE_BYTES - 1]; + } + phaseChecksum += hold(textures, TEXTURE_BYTES); + long heldKb = footprintKb(); + endPhase(name, "textures=" + TEXTURE_COUNT + + " liveBytes=" + ((long) TEXTURE_COUNT * TEXTURE_BYTES)); + + textures = null; + releasePhase(name, heldKb, false); + checksum = checksum * 131 + phaseChecksum; + } + + private static void beginPhase(String name) { + settle(); + mark("BASELINE", name); + mark("BEGIN", name); + } + + private static void endPhase(String name, String stats) { + mark("HELD", name); + System.out.println("ARM_STATS name=" + name + " " + stats); + } + + /** + * @param expectDrop whether this phase's memory is expected to come back. + * Only the small-object warm-up asserts on its released + * figure; the texture phases report theirs, and waiting + * out the full budget for a drop that is not expected + * there just burns wall time (15s per phase, measured). + */ + private static void releasePhase(String name, long heldKb, boolean expectDrop) { + scrubStack(SCRUB_DEPTH); + if (expectDrop) { + settleForRelease(heldKb); + } else { + settle(); + } + mark("RELEASED", name); + // Give an external sampler room to take a reading at the stamped + // instant; the final phase's RELEASED is otherwise raced by exit. + sleep(HOLD_PAUSE_MS * 2); + } + + private static void mark(String phase, String name) { + System.out.println("ARM_" + phase + " name=" + name + + " tMs=" + System.currentTimeMillis() + + " footprintKb=" + footprintKb()); + } + + /** + * This process's phys_footprint in KB. Runtime.totalMemory() reports + * physical RAM and Runtime.freeMemory() reports physical RAM minus + * phys_footprint, so the difference is the footprint itself. + */ + private static long footprintKb() { + Runtime r = Runtime.getRuntime(); + long kb = (r.totalMemory() - r.freeMemory()) / 1024; + if (trackMinFootprint && kb < minFootprintAfterDrop) { + minFootprintAfterDrop = kb; + } + return kb; + } + + /** + * Keeps {@code live} reachable for a sampleable window. The touch between + * pauses is what makes the reachability provable rather than incidental -- + * a conservative collector might keep it alive anyway, and this test must + * not depend on that. + */ + private static long hold(byte[][] live, int elementSize) { + long c = 0; + for (int r = 0; r < HOLD_ROUNDS; r++) { + sleep(HOLD_PAUSE_MS); + byte[] probe = live[(r * 7919) % live.length]; + c += probe[0] + probe[elementSize - 1]; + } + return c; + } + + /** + * Overwrite the stack region the phase just used, then settle. ParparVM + * scans thread stacks CONSERVATIVELY, so a dead slot still holding the + * address of the dropped ring keeps every object it referenced reachable -- + * the collector cannot tell a stale word from a live reference. Measured on + * Linux: without this the warm-up's 192MB was still fully resident after its + * settle and only came back during the NEXT phase, once that phase's frames + * had overwritten the words. Recursing writes fresh values over those slots + * so the drop is observable where it actually happens. + */ + /** + * Wait for the collector to give the phase's memory back, up to a bounded + * budget. Reclamation is ASYNCHRONOUS and takes a platform-dependent number + * of cycles -- a BiBOP object needs three sweeps to die, the major sweep + * that refills the free pool runs on a cadence, and cycles are paced at + * 200ms -- so waiting a fixed time measures the runner rather than the + * collector. Measured: the drop lands about 2s after the ring is dropped on + * macOS and about 7s in a Linux container, and a fixed six-round settle + * reported the Linux run as having released nothing. + * + *

Waiting for the drop we are about to assert on is deliberate and is not + * circular: the budget is finite, so a release that never happens still + * fails the assertion -- it just fails on the real behaviour rather than on + * whichever machine ran it. + */ + private static void settleForRelease(long heldKb) { + long target = (heldKb * 3) / 5; + for (int i = 0; i < SETTLE_MAX_ROUNDS; i++) { + System.gc(); + sleep(SETTLE_PAUSE_MS); + if (i + 1 >= SETTLE_MIN_ROUNDS && footprintKb() <= target) { + return; + } + } + } + + private static long scrubStack(int depth) { + long a = depth, b = depth + 1, c = depth + 2, d = depth + 3; + long e = depth + 4, f = depth + 5, g = depth + 6, h = depth + 7; + if (depth <= 0) { + return a + b + c + d + e + f + g + h; + } + return a + b + c + d + e + f + g + h + scrubStack(depth - 1); + } + + private static void sleep(long ms) { + try { + Thread.sleep(ms); + } catch (InterruptedException e) { + // Shortening a measurement window only makes the result more + // conservative; there is nothing to recover from. + } + } + + /** + * Collect until the footprint stops falling, rather than for a fixed number + * of rounds. System.gc() only sets forceGc and notifies the collector thread + * -- it does not stop the world and does not wait -- so each round is a + * request plus a pause, and the loop exits when a round stops buying + * anything. Bounded so a platform that never reports a falling footprint + * cannot hang the probe. + */ + private static void settle() { + long prev = footprintKb(); + int stable = 0; + for (int i = 0; i < SETTLE_PLAIN_MAX_ROUNDS; i++) { + System.gc(); + sleep(SETTLE_PAUSE_MS); + long now = footprintKb(); + stable = (now + SETTLE_STABLE_KB >= prev) ? stable + 1 : 0; + prev = now; + if (i + 1 >= SETTLE_PLAIN_MIN_ROUNDS && stable >= SETTLE_STABLE_STREAK) { + return; + } + } + } +}