Skip to content

Add GPU memory allocation cache to OpenCL backend. #4180

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
brotherofken wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Add GPU memory allocation cache to OpenCL backend. #4180

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
a48e3ed
Add GPU memory allocation cache to OpenCL backend (ported from CUDA b…
brotherofken Aug 27, 2019
c276049
Remove unused comment and include.
brotherofken Aug 27, 2019
29cb871
Move cache handling code to separate functions. Fix opencl_allocation…
brotherofken Aug 28, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
201 changes: 178 additions & 23 deletions src/runtime/opencl.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
#include "device_buffer_utils.h"
#include "device_interface.h"
#include "printer.h"

#include "mini_cl.h"
#include "scoped_mutex_lock.h"

#define INLINE inline __attribute__((always_inline))

Expand Down Expand Up @@ -216,6 +216,16 @@ struct device_handle {
cl_mem mem;
};

// A free list, used when allocations are being cached.
WEAK struct FreeListItem {
device_handle *ptr;
cl_context ctx;
cl_command_queue stream;
size_t size;
FreeListItem *next;
} *free_list = 0;
WEAK halide_mutex free_list_lock;

// Structure to hold the state of a module attached to the context.
// Also used as a linked-list to keep track of all the different
// modules that are attached to a context in order to release them all
Expand Down Expand Up @@ -477,6 +487,131 @@ WEAK int create_opencl_context(void *user_context, cl_context *ctx, cl_command_q

}}}} // namespace Halide::Runtime::Internal::OpenCL

WEAK int halide_opencl_release_unused_device_allocations(void *user_context) {
FreeListItem *to_free;
{
ScopedMutexLock lock(&free_list_lock);
to_free = free_list;
free_list = NULL;
}
while (to_free) {
debug(user_context) << " clReleaseMemObject " << (void *)to_free->ptr->mem << "\n";
cl_int err = clReleaseMemObject((cl_mem)to_free->ptr->mem);
free(to_free->ptr);

if (err != CL_SUCCESS) {
debug(user_context) << " Error during clReleaseMemObject. Error code: " << err << "\n";
}
FreeListItem *next = to_free->next;
free(to_free);
to_free = next;
}
return 0;
}

namespace Halide {
namespace Runtime {
namespace Internal {

WEAK halide_device_allocation_pool opencl_allocation_pool = {NULL, NULL};

__attribute__((constructor))
WEAK void register_opencl_allocation_pool() {
opencl_allocation_pool.release_unused = &halide_opencl_release_unused_device_allocations;
halide_register_device_allocation_pool(&opencl_allocation_pool);
}

__attribute__((always_inline))
WEAK uint64_t quantize_allocation_size(uint64_t sz) {
int z = __builtin_clzll(sz);
if (z < 60) {
sz--;
sz = sz >> (60 - z);
sz++;
sz = sz << (60 - z);
}
return sz;
}

void cache_allocation(void *user_context, ClContext& cl_ctx, halide_buffer_t *buf) {
cl_mem dev_ptr = ((device_handle *)buf->device)->mem;

debug(user_context) << " caching allocation for later use: " << (void *)(dev_ptr) << "\n";
FreeListItem *item = (FreeListItem *)malloc(sizeof(FreeListItem));
item->ctx = cl_ctx.context;
item->size = quantize_allocation_size(buf->size_in_bytes());
item->ptr = (device_handle*)buf->device;
item->stream = cl_ctx.cmd_queue;
{
ScopedMutexLock lock(&free_list_lock);
item->next = free_list;
free_list = item;
}
}

device_handle *retrieve_allocation_from_cache(void *user_context, ClContext& cl_ctx, const size_t size) {
ScopedMutexLock lock(&free_list_lock);
// Best-fit allocation. There are three tunable constants
// here. A bucket is claimed if the size requested is at least
// 7/8 of the size of the bucket. We keep at most 32 unused
// allocations. We round up each allocation size to its top 4
// most significant bits (see quantize_allocation_size).
device_handle *result = NULL;

FreeListItem *best = NULL, *item = free_list;
FreeListItem **best_prev = NULL, **prev_ptr = &free_list;
FreeListItem *to_free = NULL;

int depth = 0;
while (item) {
if ((size <= item->size) && // Fits
(size >= (item->size / 8) * 7) && // Not too much slop
(cl_ctx.context == item->ctx) && // Same cuda context
(cl_ctx.cmd_queue == item->stream) && // Can only safely re-use on the same stream on which it was freed
((best == NULL) || (best->size > item->size))) { // Better than previous best fit
best = item;
best_prev = prev_ptr;
prev_ptr = &item->next;
item = item->next;
} else if (depth > 32) {
// Allocations after here have not been used for a
// long time. Just detach the rest of the free list
// and defer the actual cuMemFree calls until after we
// release the free_list_lock.
to_free = item;
*prev_ptr = NULL;
item = NULL;
break;
} else {
prev_ptr = &item->next;
item = item->next;
}
depth++;
}

if (best) {
result = best->ptr;
*best_prev = best->next;
free(best);
}

while (to_free) {
FreeListItem *next = to_free->next;
debug(user_context) << " clReleaseMemObject from allocation cache" << (void *)to_free->ptr->mem << "\n";
cl_int err = clReleaseMemObject((cl_mem)to_free->ptr->mem);
free(to_free->ptr);
if (err != CL_SUCCESS) {
debug(user_context) << " Error during clReleaseMemObject. Error code: " << err << "\n";
}
free(to_free);
to_free = next;
}

return result;
}

}}}

extern "C" {

WEAK int halide_opencl_device_free(void *user_context, halide_buffer_t* buf) {
Expand Down Expand Up @@ -504,11 +639,18 @@ WEAK int halide_opencl_device_free(void *user_context, halide_buffer_t* buf) {
#endif

halide_assert(user_context, validate_device_pointer(user_context, buf));
debug(user_context) << " clReleaseMemObject " << (void *)dev_ptr << "\n";
cl_int result = clReleaseMemObject((cl_mem)dev_ptr);
// If clReleaseMemObject fails, it is unlikely to succeed in a later call, so
// we just end our reference to it regardless.
free((device_handle *)buf->device);

cl_int result = CL_SUCCESS;
if (halide_can_reuse_device_allocations(user_context)) {
cache_allocation(user_context, ctx, buf);
} else {
debug(user_context) << " clReleaseMemObject " << (void *)dev_ptr << "\n";
result = clReleaseMemObject((cl_mem)dev_ptr);
// If clReleaseMemObject fails, it is unlikely to succeed in a later call, so
// we just end our reference to it regardless.
free((device_handle *)buf->device);
}

buf->device = 0;
buf->device_interface->impl->release_module();
buf->device_interface = NULL;
Expand Down Expand Up @@ -683,6 +825,9 @@ WEAK int halide_opencl_device_release(void *user_context) {
err = clFinish(q);
halide_assert(user_context, err == CL_SUCCESS);

// Dump the contents of the free list, ignoring errors.
halide_opencl_release_unused_device_allocations(user_context);

// Unload the modules attached to this context. Note that the list
// nodes themselves are not freed, only the program objects are
// released. Subsequent calls to halide_init_kernels might re-create
Expand Down Expand Up @@ -729,6 +874,10 @@ WEAK int halide_opencl_device_malloc(void *user_context, halide_buffer_t* buf) {
}

size_t size = buf->size_in_bytes();
if (halide_can_reuse_device_allocations(user_context)) {
size = quantize_allocation_size(size);
}

halide_assert(user_context, size != 0);
if (buf->device) {
halide_assert(user_context, validate_device_pointer(user_context, buf, size));
Expand All @@ -739,33 +888,39 @@ WEAK int halide_opencl_device_malloc(void *user_context, halide_buffer_t* buf) {
halide_assert(user_context, buf->dim[i].stride >= 0);
}


debug(user_context) << " allocating " << *buf << "\n";

#ifdef DEBUG_RUNTIME
uint64_t t_before = halide_current_time_ns(user_context);
#endif

device_handle *dev_handle = (device_handle *)malloc(sizeof(device_handle));
if (dev_handle == NULL) {
return CL_OUT_OF_HOST_MEMORY;
device_handle *dev_handle = NULL;
if (halide_can_reuse_device_allocations(user_context)) {
dev_handle = retrieve_allocation_from_cache(user_context, ctx, size);
}

cl_int err;
debug(user_context) << " clCreateBuffer -> " << (int)size << " ";
cl_mem dev_ptr = clCreateBuffer(ctx.context, CL_MEM_READ_WRITE, size, NULL, &err);
if (err != CL_SUCCESS || dev_ptr == 0) {
debug(user_context) << get_opencl_error_name(err) << "\n";
error(user_context) << "CL: clCreateBuffer failed: "
<< get_opencl_error_name(err);
free(dev_handle);
return err;
} else {
debug(user_context) << (void *)dev_ptr << " device_handle: " << dev_handle << "\n";
if (!dev_handle) {
dev_handle = (device_handle *) malloc(sizeof(device_handle));
if (dev_handle == NULL) {
return CL_OUT_OF_HOST_MEMORY;
}

cl_int err;
debug(user_context) << " clCreateBuffer -> " << (int) size << " ";
cl_mem dev_ptr = clCreateBuffer(ctx.context, CL_MEM_READ_WRITE, size, NULL, &err);
if (err != CL_SUCCESS || dev_ptr == 0) {
debug(user_context) << get_opencl_error_name(err) << "\n";
error(user_context) << "CL: clCreateBuffer failed: "
<< get_opencl_error_name(err);
free(dev_handle);
return err;
} else {
debug(user_context) << (void *) dev_ptr << " device_handle: " << dev_handle << "\n";
}
dev_handle->mem = dev_ptr;
dev_handle->offset = 0;
}

dev_handle->mem = dev_ptr;
dev_handle->offset = 0;
buf->device = (uint64_t)dev_handle;
buf->device_interface = &opencl_device_interface;
buf->device_interface->impl->use_module();
Expand Down