diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..47a38a9 --- /dev/null +++ b/.clang-format @@ -0,0 +1,2 @@ +DisableFormat: true +SortIncludes: Never diff --git a/.omc/project-memory.json b/.omc/project-memory.json new file mode 100644 index 0000000..f5c4334 --- /dev/null +++ b/.omc/project-memory.json @@ -0,0 +1,172 @@ +{ + "version": "1.0.0", + "lastScanned": 1770613288440, + "projectRoot": "/home/dylee/workspace/sdk_release/driver", + "techStack": { + "languages": [ + { + "name": "C/C++", + "version": null, + "confidence": "high", + "markers": [ + "Makefile" + ] + } + ], + "frameworks": [], + "packageManager": null, + "runtime": null + }, + "build": { + "buildCommand": "make build", + "testCommand": "make test", + "lintCommand": null, + "devCommand": null, + "scripts": {} + }, + "conventions": { + "namingStyle": null, + "importStyle": null, + "testPattern": null, + "fileOrganization": null + }, + "structure": { + "isMonorepo": false, + "workspaces": [], + "mainDirectories": [], + "gitBranches": { + "defaultBranch": "main", + "branchingStrategy": null + } + }, + "customNotes": [], + "directoryMap": { + "config": { + "path": "config", + "purpose": "Configuration files", + "fileCount": 2, + "lastAccessed": 1770613288438, + "keyFiles": [ + "99-xcena_set_devdax_perm.rules", + "xcena_set_devdax_perm" + ] + } + }, + "hotPaths": [ + { + "path": "mx_dma.h", + "accessCount": 21, + "lastAccessed": 1770615301695, + "type": "file" + }, + { + "path": "core_v2.c", + "accessCount": 21, + "lastAccessed": 1770615395396, + "type": "file" + }, + { + "path": "", + "accessCount": 20, + "lastAccessed": 1770613668536, + "type": "directory" + }, + { + "path": "core_v1.c", + "accessCount": 17, + "lastAccessed": 1770615295430, + "type": "file" + }, + { + "path": "init.c", + "accessCount": 11, + "lastAccessed": 1770615296090, + "type": "file" + }, + { + "path": "transfer.c", + "accessCount": 11, + "lastAccessed": 1770615415367, + "type": "file" + }, + { + "path": "Makefile", + "accessCount": 9, + "lastAccessed": 1770614878669, + "type": "file" + }, + { + "path": "helper.c", + "accessCount": 9, + "lastAccessed": 1770615347458, + "type": "file" + }, + { + "path": "core_common.c", + "accessCount": 8, + "lastAccessed": 1770614894521, + "type": "file" + }, + { + "path": "ioctl.c", + "accessCount": 8, + "lastAccessed": 1770615296749, + "type": "file" + }, + { + "path": "mbox.c", + "accessCount": 8, + "lastAccessed": 1770615297067, + "type": "file" + }, + { + "path": "fops.c", + "accessCount": 7, + "lastAccessed": 1770615296373, + "type": "file" + }, + { + "path": "install.sh", + "accessCount": 6, + "lastAccessed": 1770614891369, + "type": "file" + }, + { + "path": "uninstall.sh", + "accessCount": 5, + "lastAccessed": 1770614891580, + "type": "file" + }, + { + "path": "LICENSE", + "accessCount": 1, + "lastAccessed": 1770613329122, + "type": "file" + }, + { + "path": "config/xcena_set_devdax_perm", + "accessCount": 1, + "lastAccessed": 1770613333326, + "type": "file" + }, + { + "path": "config/99-xcena_set_devdax_perm.rules", + "accessCount": 1, + "lastAccessed": 1770613333975, + "type": "file" + }, + { + "path": "CLAUDE.md", + "accessCount": 1, + "lastAccessed": 1770613492288, + "type": "file" + }, + { + "path": ".clang-format", + "accessCount": 1, + "lastAccessed": 1770614505666, + "type": "file" + } + ], + "userDirectives": [] +} \ No newline at end of file diff --git a/.omc/sessions/1568bcfb-1519-4dc3-a26e-f2a24a6b9c58.json b/.omc/sessions/1568bcfb-1519-4dc3-a26e-f2a24a6b9c58.json new file mode 100644 index 0000000..071eb24 --- /dev/null +++ b/.omc/sessions/1568bcfb-1519-4dc3-a26e-f2a24a6b9c58.json @@ -0,0 +1,8 @@ +{ + "session_id": "1568bcfb-1519-4dc3-a26e-f2a24a6b9c58", + "ended_at": "2026-02-09T05:27:24.064Z", + "reason": "clear", + "agents_spawned": 0, + "agents_completed": 0, + "modes_used": [] +} \ No newline at end of file diff --git a/.omc/sessions/6fd883fb-3c44-4079-a450-a4e2bb221724.json b/.omc/sessions/6fd883fb-3c44-4079-a450-a4e2bb221724.json new file mode 100644 index 0000000..7e68716 --- /dev/null +++ b/.omc/sessions/6fd883fb-3c44-4079-a450-a4e2bb221724.json @@ -0,0 +1,8 @@ +{ + "session_id": "6fd883fb-3c44-4079-a450-a4e2bb221724", + "ended_at": "2026-02-09T05:37:28.857Z", + "reason": "prompt_input_exit", + "agents_spawned": 0, + "agents_completed": 0, + "modes_used": [] +} \ No newline at end of file diff --git a/.omc/sessions/75b89100-9a9f-47ac-bfd6-b43f8eef3573.json b/.omc/sessions/75b89100-9a9f-47ac-bfd6-b43f8eef3573.json new file mode 100644 index 0000000..e5ea15e --- /dev/null +++ b/.omc/sessions/75b89100-9a9f-47ac-bfd6-b43f8eef3573.json @@ -0,0 +1,8 @@ +{ + "session_id": "75b89100-9a9f-47ac-bfd6-b43f8eef3573", + "ended_at": "2026-02-09T05:19:22.141Z", + "reason": "prompt_input_exit", + "agents_spawned": 0, + "agents_completed": 0, + "modes_used": [] +} \ No newline at end of file diff --git a/.omc/sessions/98ac4a19-ae3f-4aa5-be2a-bb138d485e75.json b/.omc/sessions/98ac4a19-ae3f-4aa5-be2a-bb138d485e75.json new file mode 100644 index 0000000..0cdf297 --- /dev/null +++ b/.omc/sessions/98ac4a19-ae3f-4aa5-be2a-bb138d485e75.json @@ -0,0 +1,8 @@ +{ + "session_id": "98ac4a19-ae3f-4aa5-be2a-bb138d485e75", + "ended_at": "2026-02-09T05:34:49.749Z", + "reason": "clear", + "agents_spawned": 3, + "agents_completed": 3, + "modes_used": [] +} \ No newline at end of file diff --git a/.omc/sessions/a5effaac-8e3f-4cc8-9c7b-3c5b2b5061a5.json b/.omc/sessions/a5effaac-8e3f-4cc8-9c7b-3c5b2b5061a5.json new file mode 100644 index 0000000..5ca3e26 --- /dev/null +++ b/.omc/sessions/a5effaac-8e3f-4cc8-9c7b-3c5b2b5061a5.json @@ -0,0 +1,8 @@ +{ + "session_id": "a5effaac-8e3f-4cc8-9c7b-3c5b2b5061a5", + "ended_at": "2026-02-09T05:16:46.953Z", + "reason": "clear", + "agents_spawned": 6, + "agents_completed": 6, + "modes_used": [] +} \ No newline at end of file diff --git a/.omc/state/checkpoints/checkpoint-2026-02-09T05-25-38-294Z.json b/.omc/state/checkpoints/checkpoint-2026-02-09T05-25-38-294Z.json new file mode 100644 index 0000000..8c8e83e --- /dev/null +++ b/.omc/state/checkpoints/checkpoint-2026-02-09T05-25-38-294Z.json @@ -0,0 +1,11 @@ +{ + "created_at": "2026-02-09T05:25:38.294Z", + "trigger": "manual", + "active_modes": {}, + "todo_summary": { + "pending": 0, + "in_progress": 0, + "completed": 0 + }, + "wisdom_exported": false +} \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..07b7f73 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,120 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +Linux out-of-tree kernel module (`mx_dma.ko`) for XCENA MX-DMA PCI devices. Provides DMA transfer capabilities between host memory and CXL (Compute Express Link) memory devices. Supports both CXL-enabled and standalone (non-CXL) configurations. + +## Build Commands + +```bash +# Build with CXL support (default) +make + +# Build without CXL support (standalone mode) +make WO_CXL=1 + +# Clean build artifacts +make clean + +# Install (auto-detects CXL via /sys/firmware/acpi/tables/CEDT) +sudo ./install.sh + +# Uninstall +sudo ./uninstall.sh + +# Build against a specific kernel +make BUILDSYSTEM_DIR=/lib/modules//build +``` + +The module requires kernel headers at `/lib/modules/$(uname -r)/build`. There is no test suite in this repository; testing is done through userspace applications in the parent `sdk_release` repo. + +## Architecture + +### Module Structure + +The driver registers as a PCI driver for vendor `0x20A6` (XCENA). Each PCI device creates 5 character devices under `/dev/mx_dma/`: + +| Device Node | Purpose | Key Operations | +|---|---|---| +| `mx_dma{N}_data` | Bulk data DMA transfers | read/write (scatter-gather, parallel) | +| `mx_dma{N}_context` | Context/control transfers | read/write (single transfer) | +| `mx_dma{N}_ioctl` | Mailbox and control commands | ioctl | +| `mx_dma{N}_event` | MSI interrupt events | poll | +| `mx_dma{N}_bdf` | PCI BDF information | read | + +### Hardware Revision Abstraction + +The driver supports two hardware revisions selected at probe time via `pdev->revision`: + +- **Revision 1** (`core_v1.c`): Custom mailbox protocol with direct BAR MMIO. Uses 1KB (`SINGLE_DMA_SIZE = 1 << 10`) DMA granularity. SQ/CQ are MMIO-mapped mailbox regions in BAR space. +- **Revision 2** (`core_v2.c`): NVMe-like admin/IO queue model with doorbell-based submission. Uses 4KB (`SINGLE_DMA_SIZE = PAGE_SIZE`) DMA granularity. SQ/CQ are DMA-coherent host memory buffers. Admin queue sets up IO queues via create/delete commands. + +Both revisions implement `struct mx_operations` (init_queue, release_queue, create_command_sg, create_command_ctrl) registered via `register_mx_ops_v1/v2`. + +### Source File Responsibilities + +- **`init.c`** — Module init/exit, PCI probe/remove, character device creation, CXL device discovery. CXL mode uses `bus_register_notifier` on `pci_bus_type`; non-CXL mode uses standard `pci_register_driver`. +- **`fops.c`** — File operations for all 5 character device types. Routes reads/writes through `mxdma_device_prepare()` magic validation. +- **`transfer.c`** — DMA transfer lifecycle: user page pinning (`pin_user_pages_fast`), scatter-gather mapping, parallel transfer splitting, completion waiting, zombie cleanup. Module params: `timeout_ms` (default 60000), `parallel_count` (default 6). +- **`ioctl.c`** — IOCTL handlers for mailbox management (register, init, send/recv commands, read/write data). Defines `MX_IOCTL_MAGIC 'X'` with 7 ioctl commands. +- **`mbox.c`** — Mailbox ring buffer utilities (empty/full checks, index arithmetic with phase-bit wraparound). +- **`helper.c`** — Global transfer ID management via IDR with 16-bit cyclic allocation. +- **`core_v1.c` / `core_v2.c`** — Hardware-specific queue init, submit/complete handler threads, and command creation. + +### Key Data Flow + +``` +User read/write → fops.c (magic validation) + → transfer.c: alloc_mx_transfers() splits by pages (up to parallel_count) + → transfer.c: map_user_addr_to_sg() pins pages + DMA maps + → core_vN.c: create_command_sg() builds hw command with PRP/desc lists + → transfer.c: mx_transfer_queue_parallel() enqueues to io_queue + → core_vN.c: submit_handler thread pushes commands to hardware + → core_vN.c: complete_handler thread polls completions + → transfer.c: mx_transfer_wait() with interruptible timeout +``` + +### Concurrency Model + +- **submit_thread / complete_thread** — Per-device kthreads that poll SQ/CQ with `swait_queue_head` and `POLLING_INTERVAL_MSEC` (4ms) timeout. +- **sq_lock** (spinlock) — Protects the submission queue list. +- **Mailbox mutexes** — Per-mailbox mutex in `struct mx_mbox` for ioctl command serialization. +- **IDR id_lock** (spinlock) — Protects global transfer ID allocation/lookup. +- **zombie_lock** (spinlock) — Protects zombie transfer list; zombie_cleanup_thread runs with 5-minute grace period. + +### CXL vs Standalone Mode + +Controlled by `CONFIG_WO_CXL` (set via `make WO_CXL=1`): +- **CXL mode** (default): Uses PCI bus notifier to detect CXL-bound devices. Device ID derived from CXL memory device name (`mem{N}`). Global device list tracks all probed devices. +- **Standalone mode** (`WO_CXL=1`): Uses standard `pci_register_driver`. Device IDs are auto-incremented. + +### IOCTL Interface + +Magic: `'X'`, commands defined in `ioctl.c`: +- `MX_IOCTL_REGISTER_MBOX` (1) — Register SQ/CQ mailbox pair (up to 80 pairs) +- `MX_IOCTL_INIT_MBOX` (2) — Reset mailbox context +- `MX_IOCTL_SEND_CMD_WITH_DATA` (3) — Send command with optional data write +- `MX_IOCTL_RECV_CMDS` (4) — Receive commands from CQ mailbox +- `MX_IOCTL_SEND_CMDS` (5) — Batch send commands to SQ mailbox +- `MX_IOCTL_READ_DATA` (6) / `MX_IOCTL_WRITE_DATA` (7) — Direct parallel data transfers + +## Kernel Version Compatibility + +The code handles multiple kernel versions with `LINUX_VERSION_CODE` checks: +- `< 6.1.6`: `mxdma_devnode` uses `struct device *` +- `>= 6.1.6`: `mxdma_devnode` uses `const struct device *` +- `< 6.3.3`: `class_create` takes `THIS_MODULE` argument +- `>= 6.3.3`: `class_create` takes only the name +- `< 6.12.0`: `match_mem_prefix` callback uses `void *data` +- `>= 6.12.0`: `match_mem_prefix` callback uses `const void *data` + +## Critical Areas + +Changes to the following require extra care: +- **DMA mapping/unmapping** (`transfer.c`) — Must maintain proper pin/unpin and map/unmap pairing to avoid memory corruption. +- **Zombie transfer handling** — Prevents use-after-free when transfers timeout or are interrupted. +- **PRP/descriptor list construction** (`core_v1.c`, `core_v2.c`) — Linked-list DMA descriptor chains must maintain correct bus addresses. +- **Mailbox index arithmetic** (`mbox.c`) — Phase-bit wraparound logic is subtle; `depth` must be power-of-2. +- **Kernel version ifdefs** — Must be kept in sync when targeting new kernel versions. diff --git a/Makefile b/Makefile index 8f9e133..38dd0e2 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ INSTALL_MOD_PATH_ARG := $(if $(strip $(INSTALL_MOD_PATH)),INSTALL_MOD_PATH="$(IN ifneq ($(KERNELRELEASE),) obj-m += mx_dma.o - mx_dma-objs := init.o fops.o helper.o transfer.o mbox.o ioctl.o core_v1.o core_v2.o + mx_dma-objs := init.o fops.o helper.o transfer.o mbox.o ioctl.o core_common.o core_v1.o core_v2.o ifeq ($(WO_CXL),1) EXTRA_CFLAGS += -DCONFIG_WO_CXL endif diff --git a/core_common.c b/core_common.c new file mode 100644 index 0000000..d06f312 --- /dev/null +++ b/core_common.c @@ -0,0 +1,184 @@ +// SPDX-License-Identifier: + +#include "mx_dma.h" + +/******************************************************************************/ +/* Descriptor list utilities */ +/******************************************************************************/ +int mx_get_list_count(int total_desc_cnt, int descs_per_list) +{ + int list_cnt = 1; + + while (total_desc_cnt > descs_per_list) { + total_desc_cnt -= (descs_per_list - 1); + list_cnt++; + } + + return list_cnt; +} + +int mx_get_total_desc_count(struct sg_table *sgt, size_t dma_size, + bool skip_first) +{ + struct scatterlist *sg = sgt->sgl; + int total_desc_cnt = 0; + int i; + + for_each_sgtable_dma_sg(sgt, sg, i) { + int len = sg_dma_len(sg); + int desc_cnt = (len + dma_size - 1) / dma_size; + + total_desc_cnt += desc_cnt; + } + + if (skip_first) + total_desc_cnt--; + + return total_desc_cnt; +} + +uint64_t mx_desc_list_init(struct mx_pci_dev *mx_pdev, + struct mx_transfer *transfer, size_t dma_size, + int descs_per_list, bool skip_first_entry) +{ + struct sg_table *sgt = &transfer->sgt; + struct scatterlist *sg = sgt->sgl; + uint64_t *desc; + int total_desc_cnt, list_cnt, list_idx, desc_idx; + int ret; + int i; + + total_desc_cnt = mx_get_total_desc_count(sgt, dma_size, skip_first_entry); + list_cnt = mx_get_list_count(total_desc_cnt, descs_per_list); + ret = desc_list_alloc(mx_pdev, transfer, list_cnt); + if (ret) { + pr_warn("Failed to desc_list_alloc (err=%d)\n", ret); + return 0; + } + + list_idx = 0; + desc_idx = 0; + desc = (uint64_t *)transfer->desc_list_va[list_idx]; + + for_each_sgtable_dma_sg(sgt, sg, i) { + dma_addr_t dma_addr = sg_dma_address(sg); + ssize_t dma_len = sg_dma_len(sg); + ssize_t offset = sg->offset; + ssize_t len = dma_size; + + if (offset) { + ssize_t tmp = (PAGE_SIZE - offset) & (dma_size - 1); + if (tmp != 0) + len = tmp; + } + + if (skip_first_entry && i == 0) { + dma_addr += len; + dma_len -= len; + len = min_t(ssize_t, dma_len, dma_size); + } + + while (dma_len > 0) { + if (desc_idx == descs_per_list - 1 && total_desc_cnt > 1) { + desc[desc_idx] = (uint64_t)transfer->desc_list_ba[++list_idx]; + desc = (uint64_t *)transfer->desc_list_va[list_idx]; + desc_idx = 0; + } + + desc[desc_idx++] = dma_addr; + dma_addr += len; + dma_len -= len; + len = min_t(ssize_t, dma_len, dma_size); + total_desc_cnt--; + } + } + + return transfer->desc_list_ba[0]; +} + +/******************************************************************************/ +/* Thread helpers */ +/******************************************************************************/ +void mx_stop_queue_threads(struct mx_pci_dev *mx_pdev) +{ + int ret; + + if (!IS_ERR_OR_NULL(mx_pdev->submit_thread)) { + ret = kthread_stop(mx_pdev->submit_thread); + if (ret) + pr_err("submit_thread thread doesn't stop properly (err=%d)\n", ret); + } + + if (!IS_ERR_OR_NULL(mx_pdev->complete_thread)) { + ret = kthread_stop(mx_pdev->complete_thread); + if (ret) + pr_err("complete_thread thread doesn't stop properly (err=%d)\n", ret); + } +} + +/******************************************************************************/ +/* Unified submit/complete handlers */ +/******************************************************************************/ +int mx_submit_handler(void *arg) +{ + struct mx_queue *q = (struct mx_queue *)arg; + const struct mx_queue_ops *ops = q->ops; + struct mx_transfer *transfer, *tmp; + unsigned long flags; + + while (!kthread_should_stop()) { + __swait_event_interruptible_timeout(q->sq_wait, + !list_empty(&q->sq_list), + POLLING_INTERVAL_MSEC); + + spin_lock_irqsave(&q->sq_lock, flags); + list_for_each_entry_safe(transfer, tmp, &q->sq_list, entry) { + if (!ops->is_pushable(q)) + break; + + ops->push_command(q, transfer->command); + list_del_init(&transfer->entry); + + atomic_inc(&q->wait_count); + swake_up_one(&q->cq_wait); + } + spin_unlock_irqrestore(&q->sq_lock, flags); + + if (ops->post_submit) + ops->post_submit(q); + } + + return 0; +} + +int mx_complete_handler(void *arg) +{ + struct mx_queue *q = (struct mx_queue *)arg; + const struct mx_queue_ops *ops = q->ops; + struct mx_transfer *transfer; + int id; + uint64_t result; + + while (!kthread_should_stop()) { + __swait_event_interruptible_timeout(q->cq_wait, + atomic_read(&q->wait_count) > 0, + POLLING_INTERVAL_MSEC); + + while (ops->is_popable(q)) { + ops->pop_completion(q, &id, &result); + + transfer = find_transfer_by_id(id); + if (!transfer) + continue; + + atomic_dec(&q->wait_count); + transfer->result = result; + complete(&transfer->done); + } + + if (ops->post_complete) + ops->post_complete(q); + } + + return 0; +} diff --git a/core_v1.c b/core_v1.c index ad655ca..c191b26 100644 --- a/core_v1.c +++ b/core_v1.c @@ -101,149 +101,54 @@ static void pop_mx_command(struct mx_queue_v1 *queue, struct mx_command *comm) } /******************************************************************************/ -/* Functions for handler */ +/* Queue ops adapter for unified handlers */ /******************************************************************************/ -static int submit_handler(void *arg) +static bool v1_is_pushable(struct mx_queue *q) { - struct mx_queue_v1 *queue = (struct mx_queue_v1 *)arg; - struct mx_transfer *transfer, *tmp; - unsigned long flags; + struct mx_queue_v1 *queue = container_of(q, struct mx_queue_v1, common); - while (!kthread_should_stop()) { - __swait_event_interruptible_timeout(queue->common.sq_wait, - !list_empty(&queue->common.sq_list), - POLLING_INTERVAL_MSEC); + return is_pushable(queue); +} - spin_lock_irqsave(&queue->common.sq_lock, flags); - list_for_each_entry_safe(transfer, tmp, &queue->common.sq_list, entry) { - if (!is_pushable(queue)) - break; +static void v1_push_command(struct mx_queue *q, void *command) +{ + struct mx_queue_v1 *queue = container_of(q, struct mx_queue_v1, common); - push_mx_command(queue, (struct mx_command*)transfer->command); - list_del_init(&transfer->entry); + push_mx_command(queue, (struct mx_command *)command); +} - atomic_inc(&queue->common.wait_count); - swake_up_one(&queue->common.cq_wait); - } - spin_unlock_irqrestore(&queue->common.sq_lock, flags); - } +static bool v1_is_popable(struct mx_queue *q) +{ + struct mx_queue_v1 *queue = container_of(q, struct mx_queue_v1, common); - return 0; + return is_popable(queue); } -static int complete_handler(void *arg) +static void v1_pop_completion(struct mx_queue *q, int *out_id, uint64_t *out_result) { - struct mx_queue_v1 *queue = (struct mx_queue_v1 *)arg; - struct mx_transfer *transfer; + struct mx_queue_v1 *queue = container_of(q, struct mx_queue_v1, common); struct mx_command comm; - while (!kthread_should_stop()) { - __swait_event_interruptible_timeout(queue->common.cq_wait, - atomic_read(&queue->common.wait_count) > 0, - POLLING_INTERVAL_MSEC); - - while (is_popable(queue)) { - pop_mx_command(queue, &comm); - - transfer = find_transfer_by_id(comm.id); - if (!transfer) - continue; - - atomic_dec(&queue->common.wait_count); - transfer->result = comm.host_addr; - complete(&transfer->done); - } - } - - return 0; + pop_mx_command(queue, &comm); + *out_id = comm.id; + *out_result = comm.host_addr; } +static const struct mx_queue_ops v1_queue_ops = { + .is_pushable = v1_is_pushable, + .push_command = v1_push_command, + .post_submit = NULL, + .is_popable = v1_is_popable, + .pop_completion = v1_pop_completion, + .post_complete = NULL, +}; + /******************************************************************************/ /* Transfer */ /******************************************************************************/ #define SINGLE_DMA_SIZE (1 << 10) #define NUM_OF_DESC_PER_LIST (SINGLE_DMA_SIZE / sizeof(uint64_t)) -static int get_total_desc_count(struct mx_transfer *transfer) -{ - struct sg_table *sgt = &transfer->sgt; - struct scatterlist *sg = sgt->sgl; - int total_desc_cnt = 0; - int i; - - for_each_sgtable_dma_sg(sgt, sg, i) { - int len = sg_dma_len(sg); - int desc_cnt = (len + SINGLE_DMA_SIZE - 1) / SINGLE_DMA_SIZE; - - total_desc_cnt += desc_cnt; - } - - return total_desc_cnt; -} - -static int get_list_count(int total_desc_cnt) -{ - int list_cnt = 1; - - while (total_desc_cnt > NUM_OF_DESC_PER_LIST) { - total_desc_cnt -= (NUM_OF_DESC_PER_LIST - 1); - list_cnt++; - } - - return list_cnt; -} - -static uint64_t desc_list_init(struct mx_pci_dev *mx_pdev, struct mx_transfer *transfer) -{ - struct sg_table *sgt = &transfer->sgt; - struct scatterlist *sg = sgt->sgl; - uint64_t *desc; - int total_desc_cnt, list_cnt, list_idx, desc_idx; - int ret; - int i; - - total_desc_cnt = get_total_desc_count(transfer); - list_cnt = get_list_count(total_desc_cnt); - ret = desc_list_alloc(mx_pdev, transfer, list_cnt); - if (ret) { - pr_warn("Failed to desc_list_alloc (err=%d)\n", ret); - return 0; - } - - list_idx = 0; - desc_idx = 0; - desc = (uint64_t *)transfer->desc_list_va[list_idx]; - - for_each_sgtable_dma_sg(sgt, sg, i) { - dma_addr_t dma_addr = sg_dma_address(sg); - ssize_t dma_size = sg_dma_len(sg); - ssize_t offset = sg->offset; - ssize_t len = SINGLE_DMA_SIZE; - - if (offset) { - ssize_t tmp = (PAGE_SIZE - offset) & (SINGLE_DMA_SIZE - 1); - if (tmp != 0) { - len = tmp; - } - } - - while (dma_size > 0) { - if (desc_idx == NUM_OF_DESC_PER_LIST - 1 && total_desc_cnt > 1) { - desc[desc_idx] = (uint64_t)transfer->desc_list_ba[++list_idx]; - desc = (uint64_t *)transfer->desc_list_va[list_idx]; - desc_idx = 0; - } - - desc[desc_idx++] = dma_addr; - dma_addr += len; - dma_size -= len; - len = min_t(ssize_t, dma_size, SINGLE_DMA_SIZE); - total_desc_cnt--; - } - } - - return transfer->desc_list_ba[0]; -} static struct mx_command *alloc_mx_command(struct mx_transfer *transfer, int opcode) { @@ -289,7 +194,7 @@ static void *create_mx_command_sg(struct mx_pci_dev *mx_pdev, struct mx_transfer } } else { comm->page_mode = MXDMA_PAGE_MODE_MULTI; - comm->prp_entry1 = desc_list_init(mx_pdev, transfer); + comm->prp_entry1 = mx_desc_list_init(mx_pdev, transfer, SINGLE_DMA_SIZE, NUM_OF_DESC_PER_LIST, false); if (!comm->prp_entry1) { pr_warn("Failed to get desc_list_init\n"); kfree(comm); @@ -373,19 +278,20 @@ static int init_mx_queue(struct mx_pci_dev* mx_pdev) mx_mbox_init(&queue->cq_mbox, (uint64_t)ctx_addr, (uint64_t)data_addr, ctx); queue->common.dev = dev; + queue->common.ops = &v1_queue_ops; spin_lock_init(&queue->common.sq_lock); INIT_LIST_HEAD(&queue->common.sq_list); init_swait_queue_head(&queue->common.sq_wait); init_swait_queue_head(&queue->common.cq_wait); atomic_set(&queue->common.wait_count, 0); - mx_pdev->submit_thread = kthread_run(submit_handler, queue, "mx_submit_thd%d", mx_pdev->dev_id); + mx_pdev->submit_thread = kthread_run(mx_submit_handler, &queue->common, "mx_submit_thd%d", mx_pdev->dev_id); if (IS_ERR(mx_pdev->submit_thread)) { pr_err("Failed to create submit thread (err=%ld)\n", PTR_ERR(mx_pdev->submit_thread)); return PTR_ERR(mx_pdev->submit_thread); } - mx_pdev->complete_thread = kthread_run(complete_handler, queue, "mx_complete_thd%d", mx_pdev->dev_id); + mx_pdev->complete_thread = kthread_run(mx_complete_handler, &queue->common, "mx_complete_thd%d", mx_pdev->dev_id); if (IS_ERR(mx_pdev->complete_thread)) { pr_err("Failed to create complete thread (err=%ld)\n", PTR_ERR(mx_pdev->complete_thread)); kthread_stop(mx_pdev->submit_thread); @@ -399,21 +305,8 @@ static int init_mx_queue(struct mx_pci_dev* mx_pdev) static int release_mx_queue(struct mx_pci_dev *mx_pdev) { - int ret; - - if (!IS_ERR_OR_NULL(mx_pdev->submit_thread)) { - ret = kthread_stop(mx_pdev->submit_thread); - if (ret) - pr_err("submit_thread thread doesn't stop properly (err=%d)\n", ret); - } - - if (!IS_ERR_OR_NULL(mx_pdev->complete_thread)) { - ret = kthread_stop(mx_pdev->complete_thread); - if (ret) - pr_err("complete_thread thread doesn't stop properly (err=%d)\n", ret); - } - - return ret; + mx_stop_queue_threads(mx_pdev); + return 0; } void register_mx_ops_v1(struct mx_operations *ops) diff --git a/core_v2.c b/core_v2.c index feb68b0..15db75c 100644 --- a/core_v2.c +++ b/core_v2.c @@ -144,160 +144,68 @@ static void ring_cq_doorbell(struct mx_queue_v2 *queue) } /******************************************************************************/ -/* Functions for handler */ +/* Queue ops adapter for unified handlers */ /******************************************************************************/ -static int submit_handler(void *arg) +static bool v2_is_pushable(struct mx_queue *q) { - struct mx_queue_v2 *queue = (struct mx_queue_v2 *)arg; - struct mx_transfer *transfer, *tmp; - unsigned long flags; + struct mx_queue_v2 *queue = container_of(q, struct mx_queue_v2, common); - while (!kthread_should_stop()) { - __swait_event_interruptible_timeout(queue->common.sq_wait, - !list_empty(&queue->common.sq_list), - POLLING_INTERVAL_MSEC); - - spin_lock_irqsave(&queue->common.sq_lock, flags); - list_for_each_entry_safe(transfer, tmp, &queue->common.sq_list, entry) { - if (!is_pushable(queue)) - break; - - push_mx_command(queue, transfer->command); - list_del_init(&transfer->entry); - - atomic_inc(&queue->common.wait_count); - swake_up_one(&queue->common.cq_wait); - } - spin_unlock_irqrestore(&queue->common.sq_lock, flags); - - ring_sq_doorbell(queue); - } - - return 0; + return is_pushable(queue); } -static int complete_handler(void *arg) +static void v2_push_command(struct mx_queue *q, void *command) { - struct mx_queue_v2 *queue = (struct mx_queue_v2 *)arg; - struct mx_transfer *transfer; - struct mx_completion cmpl; - - while (!kthread_should_stop()) { - __swait_event_interruptible_timeout(queue->common.cq_wait, - atomic_read(&queue->common.wait_count) > 0, - POLLING_INTERVAL_MSEC); - - while (is_popable(queue)) { - pop_mx_completion(queue, &cmpl); - - transfer = find_transfer_by_id(cmpl.command_id); - if (!transfer) - continue; - - atomic_dec(&queue->common.wait_count); - transfer->result = cmpl.result; - complete(&transfer->done); - } + struct mx_queue_v2 *queue = container_of(q, struct mx_queue_v2, common); - ring_cq_doorbell(queue); - } - - return 0; + push_mx_command(queue, (struct mx_command *)command); } -/******************************************************************************/ -/* Transfer */ -/******************************************************************************/ -#define SINGLE_DMA_SIZE PAGE_SIZE -#define NUM_OF_DESC_PER_LIST (SINGLE_DMA_SIZE / sizeof(uint64_t)) - -static int get_total_desc_count(struct mx_transfer *transfer) +static void v2_post_submit(struct mx_queue *q) { - struct sg_table *sgt = &transfer->sgt; - struct scatterlist *sg = sgt->sgl; - int total_desc_cnt = 0; - int i; + struct mx_queue_v2 *queue = container_of(q, struct mx_queue_v2, common); - for_each_sgtable_dma_sg(sgt, sg, i) { - int len = sg_dma_len(sg); - int desc_cnt = (len + SINGLE_DMA_SIZE - 1) / SINGLE_DMA_SIZE; + ring_sq_doorbell(queue); +} - total_desc_cnt += desc_cnt; - } +static bool v2_is_popable(struct mx_queue *q) +{ + struct mx_queue_v2 *queue = container_of(q, struct mx_queue_v2, common); - return total_desc_cnt - 1; // Exclude the first PRP entry + return is_popable(queue); } -static int get_list_count(int total_desc_cnt) +static void v2_pop_completion(struct mx_queue *q, int *out_id, uint64_t *out_result) { - int list_cnt = 1; - - while (total_desc_cnt > NUM_OF_DESC_PER_LIST) { - total_desc_cnt -= (NUM_OF_DESC_PER_LIST - 1); - list_cnt++; - } + struct mx_queue_v2 *queue = container_of(q, struct mx_queue_v2, common); + struct mx_completion cmpl; - return list_cnt; + pop_mx_completion(queue, &cmpl); + *out_id = cmpl.command_id; + *out_result = cmpl.result; } -static uint64_t desc_list_init(struct mx_pci_dev *mx_pdev, struct mx_transfer *transfer) +static void v2_post_complete(struct mx_queue *q) { - struct sg_table *sgt = &transfer->sgt; - struct scatterlist *sg = sgt->sgl; - uint64_t *desc; - int total_desc_cnt, list_cnt, list_idx, desc_idx; - int ret; - int i; + struct mx_queue_v2 *queue = container_of(q, struct mx_queue_v2, common); - total_desc_cnt = get_total_desc_count(transfer); - list_cnt = get_list_count(total_desc_cnt); - ret = desc_list_alloc(mx_pdev, transfer, list_cnt); - if (ret) { - pr_warn("Failed to desc_list_alloc (err=%d)\n", ret); - return 0; - } - - list_idx = 0; - desc_idx = 0; - desc = (uint64_t *)transfer->desc_list_va[list_idx]; - - for_each_sgtable_dma_sg(sgt, sg, i) { - dma_addr_t dma_addr = sg_dma_address(sg); - ssize_t dma_size = sg_dma_len(sg); - ssize_t offset = sg->offset; - ssize_t len = SINGLE_DMA_SIZE; - - if (offset) { - ssize_t tmp = (PAGE_SIZE - offset) & (SINGLE_DMA_SIZE - 1); - if (tmp != 0) { - len = tmp; - } - } + ring_cq_doorbell(queue); +} - // First entry is handled outside of the loop - if (i == 0) { - dma_addr += len; - dma_size -= len; - len = min_t(ssize_t, dma_size, SINGLE_DMA_SIZE); - } +static const struct mx_queue_ops v2_queue_ops = { + .is_pushable = v2_is_pushable, + .push_command = v2_push_command, + .post_submit = v2_post_submit, + .is_popable = v2_is_popable, + .pop_completion = v2_pop_completion, + .post_complete = v2_post_complete, +}; - while (dma_size > 0) { - if (desc_idx == NUM_OF_DESC_PER_LIST - 1 && total_desc_cnt > 1) { - desc[desc_idx] = (uint64_t)transfer->desc_list_ba[++list_idx]; - desc = (uint64_t *)transfer->desc_list_va[list_idx]; - desc_idx = 0; - } - - desc[desc_idx++] = dma_addr; - dma_addr += len; - dma_size -= len; - len = min_t(ssize_t, dma_size, SINGLE_DMA_SIZE); - total_desc_cnt--; - } - } +/******************************************************************************/ +/* Transfer */ +/******************************************************************************/ +#define SINGLE_DMA_SIZE PAGE_SIZE +#define NUM_OF_DESC_PER_LIST (SINGLE_DMA_SIZE / sizeof(uint64_t)) - return transfer->desc_list_ba[0]; -} static struct mx_command *alloc_mx_command(struct mx_transfer *transfer, int opcode) { @@ -348,7 +256,7 @@ static void *create_mx_command_sg(struct mx_pci_dev *mx_pdev, struct mx_transfer return NULL; } } else { - comm->prp_entry2 = desc_list_init(mx_pdev, transfer); + comm->prp_entry2 = mx_desc_list_init(mx_pdev, transfer, SINGLE_DMA_SIZE, NUM_OF_DESC_PER_LIST, true); if (!comm->prp_entry2) { pr_warn("Failed to desc_list_init\n"); kfree(comm); @@ -533,18 +441,19 @@ static int configure_io_queue(struct mx_pci_dev *mx_pdev) configure_queue(mx_pdev, io_queue, cq_id); + io_queue->common.ops = &v2_queue_ops; spin_lock_init(&io_queue->common.sq_lock); INIT_LIST_HEAD(&io_queue->common.sq_list); init_swait_queue_head(&io_queue->common.sq_wait); init_swait_queue_head(&io_queue->common.cq_wait); atomic_set(&io_queue->common.wait_count, 0); - mx_pdev->submit_thread = kthread_run(submit_handler, io_queue, "mx_submit_thd%d", mx_pdev->dev_id); + mx_pdev->submit_thread = kthread_run(mx_submit_handler, &io_queue->common, "mx_submit_thd%d", mx_pdev->dev_id); if (IS_ERR(mx_pdev->submit_thread)) { pr_err("Failed to create submit thread (err=%ld)\n", PTR_ERR(mx_pdev->submit_thread)); return PTR_ERR(mx_pdev->submit_thread); } - mx_pdev->complete_thread = kthread_run(complete_handler, io_queue, "mx_complete_thd%d", mx_pdev->dev_id); + mx_pdev->complete_thread = kthread_run(mx_complete_handler, &io_queue->common, "mx_complete_thd%d", mx_pdev->dev_id); if (IS_ERR(mx_pdev->complete_thread)) { pr_err("Failed to create complete thread (err=%ld)\n", PTR_ERR(mx_pdev->complete_thread)); kthread_stop(mx_pdev->submit_thread); @@ -586,17 +495,7 @@ static int release_io_queue(struct mx_pci_dev *mx_pdev) return ret; } - if (!IS_ERR_OR_NULL(mx_pdev->submit_thread)) { - ret = kthread_stop(mx_pdev->submit_thread); - if (ret) - pr_err("submit_thread thread doesn't stop properly (err=%d)\n", ret); - } - - if (!IS_ERR_OR_NULL(mx_pdev->complete_thread)) { - ret = kthread_stop(mx_pdev->complete_thread); - if (ret) - pr_err("complete_thread thread doesn't stop properly (err=%d)\n", ret); - } + mx_stop_queue_threads(mx_pdev); return 0; } diff --git a/mx_dma.h b/mx_dma.h index dd5b4fd..2df6477 100644 --- a/mx_dma.h +++ b/mx_dma.h @@ -163,6 +163,18 @@ struct mx_char_dev { bool enabled; }; +struct mx_queue; + +struct mx_queue_ops { + bool (*is_pushable)(struct mx_queue *q); + void (*push_command)(struct mx_queue *q, void *command); + void (*post_submit)(struct mx_queue *q); + + bool (*is_popable)(struct mx_queue *q); + void (*pop_completion)(struct mx_queue *q, int *out_id, uint64_t *out_result); + void (*post_complete)(struct mx_queue *q); +}; + struct mx_queue { struct device *dev; struct list_head sq_list; @@ -170,6 +182,7 @@ struct mx_queue { atomic_t wait_count; struct swait_queue_head sq_wait; struct swait_queue_head cq_wait; + const struct mx_queue_ops *ops; }; struct mx_operations { @@ -236,6 +249,15 @@ long ioctl_to_device(struct mx_pci_dev *mx_pdev, unsigned int cmd, unsigned long int desc_list_alloc(struct mx_pci_dev *mx_pdev, struct mx_transfer *transfer, int list_cnt); +/* core_common.c */ +int mx_get_list_count(int total_desc_cnt, int descs_per_list); +int mx_get_total_desc_count(struct sg_table *sgt, size_t dma_size, bool skip_first); +uint64_t mx_desc_list_init(struct mx_pci_dev *mx_pdev, struct mx_transfer *transfer, + size_t dma_size, int descs_per_list, bool skip_first_entry); +void mx_stop_queue_threads(struct mx_pci_dev *mx_pdev); +int mx_submit_handler(void *arg); +int mx_complete_handler(void *arg); + void register_mx_ops_v1(struct mx_operations *ops); void register_mx_ops_v2(struct mx_operations *ops);