Skip to content

vulkan: add UMA dual-access mode for zero-copy CPU access on iGPUs #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
essentialols wants to merge 1 commit into from
+38 −3
Closed

vulkan: add UMA dual-access mode for zero-copy CPU access on iGPUs #1

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 38 additions & 3 deletions ggml/src/ggml-vulkan/ggml-vulkan.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -285,13 +285,16 @@ static ggml_backend_buffer_t ggml_backend_vk_buffer_type_alloc_buffer(ggml_backe
static size_t ggml_backend_vk_buffer_type_get_alignment(ggml_backend_buffer_type_t buft);
static size_t ggml_backend_vk_buffer_type_get_max_size(ggml_backend_buffer_type_t buft);
static size_t ggml_backend_vk_buffer_type_get_alloc_size(ggml_backend_buffer_type_t buft, const ggml_tensor * tensor);
static bool vk_uma_dual_access_enabled();
static bool ggml_backend_vk_buffer_type_is_host(ggml_backend_buffer_type_t buft);

static ggml_backend_buffer_type_i ggml_backend_vk_buffer_type_interface = {
/* .get_name = */ ggml_backend_vk_buffer_type_name,
/* .alloc_buffer = */ ggml_backend_vk_buffer_type_alloc_buffer,
/* .get_alignment = */ ggml_backend_vk_buffer_type_get_alignment,
/* .get_max_size = */ ggml_backend_vk_buffer_type_get_max_size,
/* .get_alloc_size = */ ggml_backend_vk_buffer_type_get_alloc_size,
/* .is_host = */ NULL,
/* .is_host = */ ggml_backend_vk_buffer_type_is_host,
};

class vk_memory_logger;
Expand Down Expand Up @@ -2164,11 +2167,36 @@ struct ggml_backend_vk_context {

static void * const vk_ptr_base = (void *)(uintptr_t) 0x1000; // NOLINT

static bool vk_uma_dual_access_checked = false;
static bool vk_uma_dual_access_val = false;
static bool vk_uma_dual_access_enabled() {
if (!vk_uma_dual_access_checked) {
vk_uma_dual_access_val = getenv("GGML_VK_UMA_DUAL_ACCESS") != nullptr;
vk_uma_dual_access_checked = true;
}
return vk_uma_dual_access_val;
}

static bool ggml_backend_vk_buffer_type_is_host(ggml_backend_buffer_type_t buft) {
if (!vk_uma_dual_access_enabled()) {
return false;
}
ggml_backend_vk_buffer_type_context * ctx = (ggml_backend_vk_buffer_type_context *)buft->context;
return ctx->device->uma;
}

static uint64_t vk_tensor_offset(const ggml_tensor * tensor) {
const void * base = vk_ptr_base;
if (vk_uma_dual_access_enabled() && tensor->buffer) {
void * buf_base = ggml_backend_buffer_get_base(tensor->buffer);
if (buf_base != vk_ptr_base) {
base = buf_base;
}
}
if (tensor->view_src) {
return (uint8_t *) tensor->view_src->data - (uint8_t *) vk_ptr_base;
return (const uint8_t *) tensor->view_src->data - (const uint8_t *) base;
}
return (uint8_t *) tensor->data - (uint8_t *) vk_ptr_base;
return (const uint8_t *) tensor->data - (const uint8_t *) base;
}

static uint32_t get_misalign_bytes(const ggml_backend_vk_context * ctx, const ggml_tensor * t)
Expand Down Expand Up @@ -15176,6 +15204,13 @@ static void ggml_backend_vk_buffer_free_buffer(ggml_backend_buffer_t buffer) {
}

static void * ggml_backend_vk_buffer_get_base(ggml_backend_buffer_t buffer) {
if (vk_uma_dual_access_enabled()) {
ggml_backend_vk_buffer_context * ctx = (ggml_backend_vk_buffer_context *)buffer->context;
auto dev = ctx->device.lock();
if (dev && dev->uma && ctx->dev_buffer && ctx->dev_buffer->ptr) {
return ctx->dev_buffer->ptr;
}
}
return vk_ptr_base;

UNUSED(buffer);
Expand Down