Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .clang-format
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
DisableFormat: true
SortIncludes: Never
172 changes: 172 additions & 0 deletions .omc/project-memory.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,172 @@
{
"version": "1.0.0",
"lastScanned": 1770613288440,
"projectRoot": "/home/dylee/workspace/sdk_release/driver",
"techStack": {
"languages": [
{
"name": "C/C++",
"version": null,
"confidence": "high",
"markers": [
"Makefile"
]
}
],
"frameworks": [],
"packageManager": null,
"runtime": null
},
"build": {
"buildCommand": "make build",
"testCommand": "make test",
"lintCommand": null,
"devCommand": null,
"scripts": {}
},
"conventions": {
"namingStyle": null,
"importStyle": null,
"testPattern": null,
"fileOrganization": null
},
"structure": {
"isMonorepo": false,
"workspaces": [],
"mainDirectories": [],
"gitBranches": {
"defaultBranch": "main",
"branchingStrategy": null
}
},
"customNotes": [],
"directoryMap": {
"config": {
"path": "config",
"purpose": "Configuration files",
"fileCount": 2,
"lastAccessed": 1770613288438,
"keyFiles": [
"99-xcena_set_devdax_perm.rules",
"xcena_set_devdax_perm"
]
}
},
"hotPaths": [
{
"path": "mx_dma.h",
"accessCount": 21,
"lastAccessed": 1770615301695,
"type": "file"
},
{
"path": "core_v2.c",
"accessCount": 21,
"lastAccessed": 1770615395396,
"type": "file"
},
{
"path": "",
"accessCount": 20,
"lastAccessed": 1770613668536,
"type": "directory"
},
{
"path": "core_v1.c",
"accessCount": 17,
"lastAccessed": 1770615295430,
"type": "file"
},
{
"path": "init.c",
"accessCount": 11,
"lastAccessed": 1770615296090,
"type": "file"
},
{
"path": "transfer.c",
"accessCount": 11,
"lastAccessed": 1770615415367,
"type": "file"
},
{
"path": "Makefile",
"accessCount": 9,
"lastAccessed": 1770614878669,
"type": "file"
},
{
"path": "helper.c",
"accessCount": 9,
"lastAccessed": 1770615347458,
"type": "file"
},
{
"path": "core_common.c",
"accessCount": 8,
"lastAccessed": 1770614894521,
"type": "file"
},
{
"path": "ioctl.c",
"accessCount": 8,
"lastAccessed": 1770615296749,
"type": "file"
},
{
"path": "mbox.c",
"accessCount": 8,
"lastAccessed": 1770615297067,
"type": "file"
},
{
"path": "fops.c",
"accessCount": 7,
"lastAccessed": 1770615296373,
"type": "file"
},
{
"path": "install.sh",
"accessCount": 6,
"lastAccessed": 1770614891369,
"type": "file"
},
{
"path": "uninstall.sh",
"accessCount": 5,
"lastAccessed": 1770614891580,
"type": "file"
},
{
"path": "LICENSE",
"accessCount": 1,
"lastAccessed": 1770613329122,
"type": "file"
},
{
"path": "config/xcena_set_devdax_perm",
"accessCount": 1,
"lastAccessed": 1770613333326,
"type": "file"
},
{
"path": "config/99-xcena_set_devdax_perm.rules",
"accessCount": 1,
"lastAccessed": 1770613333975,
"type": "file"
},
{
"path": "CLAUDE.md",
"accessCount": 1,
"lastAccessed": 1770613492288,
"type": "file"
},
{
"path": ".clang-format",
"accessCount": 1,
"lastAccessed": 1770614505666,
"type": "file"
}
],
"userDirectives": []
}
8 changes: 8 additions & 0 deletions .omc/sessions/1568bcfb-1519-4dc3-a26e-f2a24a6b9c58.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
{
"session_id": "1568bcfb-1519-4dc3-a26e-f2a24a6b9c58",
"ended_at": "2026-02-09T05:27:24.064Z",
"reason": "clear",
"agents_spawned": 0,
"agents_completed": 0,
"modes_used": []
}
8 changes: 8 additions & 0 deletions .omc/sessions/6fd883fb-3c44-4079-a450-a4e2bb221724.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
{
"session_id": "6fd883fb-3c44-4079-a450-a4e2bb221724",
"ended_at": "2026-02-09T05:37:28.857Z",
"reason": "prompt_input_exit",
"agents_spawned": 0,
"agents_completed": 0,
"modes_used": []
}
8 changes: 8 additions & 0 deletions .omc/sessions/75b89100-9a9f-47ac-bfd6-b43f8eef3573.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
{
"session_id": "75b89100-9a9f-47ac-bfd6-b43f8eef3573",
"ended_at": "2026-02-09T05:19:22.141Z",
"reason": "prompt_input_exit",
"agents_spawned": 0,
"agents_completed": 0,
"modes_used": []
}
8 changes: 8 additions & 0 deletions .omc/sessions/98ac4a19-ae3f-4aa5-be2a-bb138d485e75.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
{
"session_id": "98ac4a19-ae3f-4aa5-be2a-bb138d485e75",
"ended_at": "2026-02-09T05:34:49.749Z",
"reason": "clear",
"agents_spawned": 3,
"agents_completed": 3,
"modes_used": []
}
8 changes: 8 additions & 0 deletions .omc/sessions/a5effaac-8e3f-4cc8-9c7b-3c5b2b5061a5.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
{
"session_id": "a5effaac-8e3f-4cc8-9c7b-3c5b2b5061a5",
"ended_at": "2026-02-09T05:16:46.953Z",
"reason": "clear",
"agents_spawned": 6,
"agents_completed": 6,
"modes_used": []
}
11 changes: 11 additions & 0 deletions .omc/state/checkpoints/checkpoint-2026-02-09T05-25-38-294Z.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{
"created_at": "2026-02-09T05:25:38.294Z",
"trigger": "manual",
"active_modes": {},
"todo_summary": {
"pending": 0,
"in_progress": 0,
"completed": 0
},
"wisdom_exported": false
}
120 changes: 120 additions & 0 deletions CLAUDE.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
# CLAUDE.md

This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.

## Project Overview

Linux out-of-tree kernel module (`mx_dma.ko`) for XCENA MX-DMA PCI devices. Provides DMA transfer capabilities between host memory and CXL (Compute Express Link) memory devices. Supports both CXL-enabled and standalone (non-CXL) configurations.

## Build Commands

```bash
# Build with CXL support (default)
make

# Build without CXL support (standalone mode)
make WO_CXL=1

# Clean build artifacts
make clean

# Install (auto-detects CXL via /sys/firmware/acpi/tables/CEDT)
sudo ./install.sh

# Uninstall
sudo ./uninstall.sh

# Build against a specific kernel
make BUILDSYSTEM_DIR=/lib/modules/<version>/build
```

The module requires kernel headers at `/lib/modules/$(uname -r)/build`. There is no test suite in this repository; testing is done through userspace applications in the parent `sdk_release` repo.

## Architecture

### Module Structure

The driver registers as a PCI driver for vendor `0x20A6` (XCENA). Each PCI device creates 5 character devices under `/dev/mx_dma/`:

| Device Node | Purpose | Key Operations |
|---|---|---|
| `mx_dma{N}_data` | Bulk data DMA transfers | read/write (scatter-gather, parallel) |
| `mx_dma{N}_context` | Context/control transfers | read/write (single transfer) |
| `mx_dma{N}_ioctl` | Mailbox and control commands | ioctl |
| `mx_dma{N}_event` | MSI interrupt events | poll |
| `mx_dma{N}_bdf` | PCI BDF information | read |

### Hardware Revision Abstraction

The driver supports two hardware revisions selected at probe time via `pdev->revision`:

- **Revision 1** (`core_v1.c`): Custom mailbox protocol with direct BAR MMIO. Uses 1KB (`SINGLE_DMA_SIZE = 1 << 10`) DMA granularity. SQ/CQ are MMIO-mapped mailbox regions in BAR space.
- **Revision 2** (`core_v2.c`): NVMe-like admin/IO queue model with doorbell-based submission. Uses 4KB (`SINGLE_DMA_SIZE = PAGE_SIZE`) DMA granularity. SQ/CQ are DMA-coherent host memory buffers. Admin queue sets up IO queues via create/delete commands.

Both revisions implement `struct mx_operations` (init_queue, release_queue, create_command_sg, create_command_ctrl) registered via `register_mx_ops_v1/v2`.

### Source File Responsibilities

- **`init.c`** — Module init/exit, PCI probe/remove, character device creation, CXL device discovery. CXL mode uses `bus_register_notifier` on `pci_bus_type`; non-CXL mode uses standard `pci_register_driver`.
- **`fops.c`** — File operations for all 5 character device types. Routes reads/writes through `mxdma_device_prepare()` magic validation.
- **`transfer.c`** — DMA transfer lifecycle: user page pinning (`pin_user_pages_fast`), scatter-gather mapping, parallel transfer splitting, completion waiting, zombie cleanup. Module params: `timeout_ms` (default 60000), `parallel_count` (default 6).
- **`ioctl.c`** — IOCTL handlers for mailbox management (register, init, send/recv commands, read/write data). Defines `MX_IOCTL_MAGIC 'X'` with 7 ioctl commands.
- **`mbox.c`** — Mailbox ring buffer utilities (empty/full checks, index arithmetic with phase-bit wraparound).
- **`helper.c`** — Global transfer ID management via IDR with 16-bit cyclic allocation.
- **`core_v1.c` / `core_v2.c`** — Hardware-specific queue init, submit/complete handler threads, and command creation.

### Key Data Flow

```
User read/write → fops.c (magic validation)
→ transfer.c: alloc_mx_transfers() splits by pages (up to parallel_count)
→ transfer.c: map_user_addr_to_sg() pins pages + DMA maps
→ core_vN.c: create_command_sg() builds hw command with PRP/desc lists
→ transfer.c: mx_transfer_queue_parallel() enqueues to io_queue
→ core_vN.c: submit_handler thread pushes commands to hardware
→ core_vN.c: complete_handler thread polls completions
→ transfer.c: mx_transfer_wait() with interruptible timeout
```

### Concurrency Model

- **submit_thread / complete_thread** — Per-device kthreads that poll SQ/CQ with `swait_queue_head` and `POLLING_INTERVAL_MSEC` (4ms) timeout.
- **sq_lock** (spinlock) — Protects the submission queue list.
- **Mailbox mutexes** — Per-mailbox mutex in `struct mx_mbox` for ioctl command serialization.
- **IDR id_lock** (spinlock) — Protects global transfer ID allocation/lookup.
- **zombie_lock** (spinlock) — Protects zombie transfer list; zombie_cleanup_thread runs with 5-minute grace period.

### CXL vs Standalone Mode

Controlled by `CONFIG_WO_CXL` (set via `make WO_CXL=1`):
- **CXL mode** (default): Uses PCI bus notifier to detect CXL-bound devices. Device ID derived from CXL memory device name (`mem{N}`). Global device list tracks all probed devices.
- **Standalone mode** (`WO_CXL=1`): Uses standard `pci_register_driver`. Device IDs are auto-incremented.

### IOCTL Interface

Magic: `'X'`, commands defined in `ioctl.c`:
- `MX_IOCTL_REGISTER_MBOX` (1) — Register SQ/CQ mailbox pair (up to 80 pairs)
- `MX_IOCTL_INIT_MBOX` (2) — Reset mailbox context
- `MX_IOCTL_SEND_CMD_WITH_DATA` (3) — Send command with optional data write
- `MX_IOCTL_RECV_CMDS` (4) — Receive commands from CQ mailbox
- `MX_IOCTL_SEND_CMDS` (5) — Batch send commands to SQ mailbox
- `MX_IOCTL_READ_DATA` (6) / `MX_IOCTL_WRITE_DATA` (7) — Direct parallel data transfers

## Kernel Version Compatibility

The code handles multiple kernel versions with `LINUX_VERSION_CODE` checks:
- `< 6.1.6`: `mxdma_devnode` uses `struct device *`
- `>= 6.1.6`: `mxdma_devnode` uses `const struct device *`
- `< 6.3.3`: `class_create` takes `THIS_MODULE` argument
- `>= 6.3.3`: `class_create` takes only the name
- `< 6.12.0`: `match_mem_prefix` callback uses `void *data`
- `>= 6.12.0`: `match_mem_prefix` callback uses `const void *data`

## Critical Areas

Changes to the following require extra care:
- **DMA mapping/unmapping** (`transfer.c`) — Must maintain proper pin/unpin and map/unmap pairing to avoid memory corruption.
- **Zombie transfer handling** — Prevents use-after-free when transfers timeout or are interrupted.
- **PRP/descriptor list construction** (`core_v1.c`, `core_v2.c`) — Linked-list DMA descriptor chains must maintain correct bus addresses.
- **Mailbox index arithmetic** (`mbox.c`) — Phase-bit wraparound logic is subtle; `depth` must be power-of-2.
- **Kernel version ifdefs** — Must be kept in sync when targeting new kernel versions.
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ INSTALL_MOD_PATH_ARG := $(if $(strip $(INSTALL_MOD_PATH)),INSTALL_MOD_PATH="$(IN

ifneq ($(KERNELRELEASE),)
obj-m += mx_dma.o
mx_dma-objs := init.o fops.o helper.o transfer.o mbox.o ioctl.o core_v1.o core_v2.o
mx_dma-objs := init.o fops.o helper.o transfer.o mbox.o ioctl.o core_common.o core_v1.o core_v2.o
ifeq ($(WO_CXL),1)
EXTRA_CFLAGS += -DCONFIG_WO_CXL
endif
Expand Down
Loading