freywar/freywaros
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add common way to access bit ranges, log all NVMe devices on startup

Browse Source
This commit is contained in:
freywar
2026-07-27 20:45:00 +03:00
parent 572cb7647f
commit a2128de3e2
5 changed files with 134 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/kernel/nvme.c
+54 -29
View File
@@ -8,6 +8,9 @@
#define SECTOR_SIZE 512
#define NVME_PCI_CLASS 0x01
#define NVME_PCI_SUBCLASS 0x08
typedef volatile struct __attribute__((packed)) {
uint64_t cap;
uint32_t vs;
@@ -22,7 +25,24 @@ typedef volatile struct __attribute__((packed)) {
uint64_t acq;
} nvme_regs_t;
#define NVME_REGS_NVME_CS_SUPPORTED(r) BITS_R(r->cap, 37, 37)
#define NVME_REGS_MQES(r) BITS_R(r->cap, 15, 0)
#define NVME_REGS_IOCQES_R(r) BITS_R(r->cc, 23, 20)
#define NVME_REGS_IOCQES_W(r, v) BITS_W(r->cc, 23, 20, v)
#define NVME_REGS_IOSQES_R(r) BITS_R(r->cc, 19, 16)
#define NVME_REGS_IOSQES_W(r, v) BITS_W(r->cc, 19, 16, v)
#define NVME_REGS_CSS_R(r) BITS_R(r->cc, 6, 4)
#define NVME_REGS_CSS_W(r, v) BITS_W(r->cc, 6, 4, v)
#define NVME_REGS_EN_R(r) BITS_R(r->cc, 0, 0)
#define NVME_REGS_EN_W(r, v) BITS_W(r->cc, 0, 0, v)
#define NVME_REGS_RDY(r) BITS_R(r->csts, 0, 0)
#define NVME_REGS_ACQS_R(r) BITS_R(r->aqa, 27, 16)
#define NVME_REGS_ACQS_W(r, v) BITS_W(r->aqa, 27, 16, v)
#define NVME_REGS_ASQS_R(r) BITS_R(r->aqa, 11, 0)
#define NVME_REGS_ASQS_W(r, v) BITS_W(r->aqa, 11, 0, v)
static nvme_regs_t *regs = (nvme_regs_t *)(KERNEL_VIRTUAL_BASE + MEMORY_SIZE);
static volatile uint32_t *db_regs = (uint32_t *)(KERNEL_VIRTUAL_BASE + MEMORY_SIZE + 0x1000); // TODO Get stride from caps.
#define QUEUE_DEPTH 2
@@ -52,12 +72,15 @@ typedef volatile struct __attribute__((packed)) {
uint16_t status;
} nvme_cqe_t;
uint8_t admin_cq_phase = 1;
#define NVME_CQE_STATUS(e) BITS_R(e->status, 15, 1)
#define NVME_CQE_P(e) BITS_R(e->status, 0, 0)
static uint8_t admin_cq_phase = 1;
static nvme_sqe_t admin_sq[QUEUE_DEPTH] __attribute__((aligned(PAGE_SIZE)));
static uint16_t admin_sq_tail = 0;
static nvme_cqe_t admin_cq[QUEUE_DEPTH] __attribute__((aligned(PAGE_SIZE)));
static uint16_t admin_cq_head = 0;
uint8_t io_cq_phase = 1;
static uint8_t io_cq_phase = 1;
static nvme_sqe_t io_sq[QUEUE_DEPTH] __attribute__((aligned(PAGE_SIZE)));
static uint16_t io_sq_tail = 0;
static nvme_cqe_t io_cq[QUEUE_DEPTH] __attribute__((aligned(PAGE_SIZE)));
@@ -78,12 +101,12 @@ static void admin_exec_sync(nvme_sqe_t *cmd) {
memory_copy(cmd, sizeof(nvme_sqe_t), &admin_sq[admin_sq_tail]);
TRACE_LN("Advancing submission doorbell...");
*(volatile uint32_t *)((uint8_t *)regs + 0x1000) = admin_sq_tail = (admin_sq_tail + 1) % QUEUE_DEPTH;
db_regs[0] = admin_sq_tail = (admin_sq_tail + 1) % QUEUE_DEPTH;
TRACE_VAL_F(admin_sq_tail, "%u");
TRACE_LN("Waiting for completion...");
nvme_cqe_t *cmp = &admin_cq[admin_cq_head];
while ((cmp->status & 1) != admin_cq_phase)
while (NVME_CQE_P(cmp) != admin_cq_phase)
;
TRACE_LN("Received completion...");
@@ -93,10 +116,10 @@ static void admin_exec_sync(nvme_sqe_t *cmd) {
TRACE_VAL_F(cmp->sqid, "%hx");
TRACE_VAL_F(cmp->cid, "%hx");
TRACE_VAL_F(cmp->status, "%hx");
ASSERT((admin_cq[admin_cq_head].status >> 1) == 0, "admin_exec_sync: NVMe command failed.");
ASSERT(NVME_CQE_STATUS(cmp) == 0, "admin_exec_sync: NVMe command failed.");
TRACE_LN("Advancing completion doorbell...");
*(volatile uint32_t *)((uint8_t *)regs + 0x1004) = admin_cq_head = (admin_cq_head + 1) % QUEUE_DEPTH; // TODO Get offset from caps.
db_regs[1] = admin_cq_head = (admin_cq_head + 1) % QUEUE_DEPTH;
admin_cq_phase ^= (admin_cq_head == 0);
TRACE_LN("Done.");
@@ -117,11 +140,11 @@ static void io_exec_sync(nvme_sqe_t *cmd) {
memory_copy(cmd, sizeof(nvme_sqe_t), &io_sq[io_sq_tail]);
TRACE_LN("Advancing submission doorbell...");
*(volatile uint32_t *)((uint8_t *)regs + 0x1008) = io_sq_tail = (io_sq_tail + 1) % QUEUE_DEPTH; // TODO Get offset from caps.
db_regs[2] = io_sq_tail = (io_sq_tail + 1) % QUEUE_DEPTH; // TODO Get offset from caps.
TRACE_LN("Waiting for completion...");
nvme_cqe_t *cmp = &io_cq[io_cq_head];
while ((cmp->status & 1) != io_cq_phase)
while (NVME_CQE_P(cmp) != io_cq_phase)
;
TRACE_LN("Received completion...");
@@ -132,10 +155,10 @@ static void io_exec_sync(nvme_sqe_t *cmd) {
TRACE_VAL_F(cmp->cid, "%hx");
TRACE_VAL_F(cmp->status, "%hx");
ASSERT((cmp->status >> 1) == 0, "io_exec_sync: NVMe command failed.");
ASSERT(NVME_CQE_STATUS(cmp) == 0, "io_exec_sync: NVMe command failed.");
TRACE_LN("Advancing completion doorbell...");
*(volatile uint32_t *)((uint8_t *)regs + 0x100C) = io_cq_head = (io_cq_head + 1) % QUEUE_DEPTH; // TODO Get offset from caps.
db_regs[3] = io_cq_head = (io_cq_head + 1) % QUEUE_DEPTH;
io_cq_phase ^= (io_cq_head == 0);
TRACE_LN("Done.");
@@ -144,27 +167,24 @@ static void io_exec_sync(nvme_sqe_t *cmd) {
void nvme_init() {
LOG_LN("Searching for suitable device...");
uint8_t found = 0;
for (uint16_t bus = 0; bus < 256 && !found; bus++) {
for (uint8_t device = 0; device < 32 && !found; device++) {
for (uint8_t function = 0; function < 8 && !found; function++) {
uint32_t id = pci_read((uint8_t)bus, device, function, 0);
if ((id & 0xFFFF) == 0xFFFF) {
for (uint16_t bus = 0; bus < PCI_MAX_BUSES && !found; bus++) {
for (uint8_t device = 0; device < PCI_MAX_DEVICES && !found; device++) {
for (uint8_t function = 0; function < PCI_MAX_FUNCTIONS && !found; function++) {
uint32_t id = pci_read((uint8_t)bus, device, function, PCI_OFFSET_DEVICE_VENDOR);
if (PCI_DEVICE(id) == PCI_DEVICE_EMPTY) {
continue;
}
uint32_t class = pci_read((uint8_t)bus, device, function, 8);
if ((class & 0xFF000000) >> 24 != 0x01 || (class & 0x00FF0000) >> 16 != 0x08) {
uint32_t class = pci_read((uint8_t)bus, device, function, PCI_OFFSET_CLASS_REVISION);
if (PCI_CLASS(class) != NVME_PCI_CLASS || PCI_CLASS_SUB(class) != NVME_PCI_SUBCLASS) {
continue;
}
pci_write(0, 4, 0, 0x04, pci_read(0, 4, 0, 0x04) | 0x06);
LOG_LN("Found device, %hx:%hx.", PCI_VENDOR(id), PCI_DEVICE(id));
uint32_t bar0 = pci_read((uint8_t)bus, device, function, 0x10) & 0xFFFFFFF0;
uint32_t bar1 = pci_read((uint8_t)bus, device, function, 0x14);
uint64_t phys = ((uint64_t)bar1 << 32) | bar0;
LOG_LN("Mapping device to memory, phys %lx <-> virt %lx...", phys, regs);
uint64_t bar = pci_bar((uint8_t)bus, device, function);
LOG_LN("Mapping device to memory, phys %lx <-> virt %lx...", bar, regs);
for (uint8_t i = 0; i < 4; i++) {
memory_page_map((void *)KERNEL_VIRTUAL_PML4, (void *)((uint8_t *)regs + i * PAGE_SIZE), (void *)((uint8_t *)phys + i * PAGE_SIZE),
memory_page_map((void *)KERNEL_VIRTUAL_PML4, (void *)((uint8_t *)regs + i * PAGE_SIZE), (void *)((uint8_t *)bar + i * PAGE_SIZE),
PAGE_WRITABLE | PAGE_PCD | PAGE_USER);
}
@@ -174,27 +194,32 @@ void nvme_init() {
}
ASSERT(found, "nvme_init: No suitable devices found.");
ASSERT(regs->cap & (1ULL << 37), "nvme_init: NVM command set not supported.");
ASSERT(NVME_REGS_NVME_CS_SUPPORTED(regs), "nvme_init: NVM command set not supported.");
ASSERT(QUEUE_DEPTH <= NVME_REGS_MQES(regs), "nvme_init: Command queues are too big.")
LOG_LN("Disabling device...");
regs->cc = 0;
TRACE_VAL_F(regs->cc, "%lx");
while (regs->csts & 1)
while (NVME_REGS_RDY(regs))
;
TRACE_VAL_F(regs->cc, "%lx");
LOG_LN("Configuring device...");
NVME_REGS_ASQS_W(regs, QUEUE_DEPTH - 1);
regs->asq = (uint64_t)(VIRT_TO_PHYS(admin_sq));
NVME_REGS_ACQS_W(regs, QUEUE_DEPTH - 1);
regs->acq = (uint64_t)(VIRT_TO_PHYS(admin_cq));
regs->aqa = (QUEUE_DEPTH - 1) << 16 | (QUEUE_DEPTH - 1);
TRACE_VAL_F(regs->asq, "%lx");
TRACE_VAL_F(regs->acq, "%lx");
TRACE_VAL_F(regs->aqa, "%lx");
LOG_LN("Enabling device...");
regs->cc = (4 << 20) | (6 << 16) | (0 << 4) | 1;
NVME_REGS_IOCQES_W(regs, 4);
NVME_REGS_IOSQES_W(regs, 6);
NVME_REGS_CSS_W(regs, 0);
NVME_REGS_EN_W(regs, 1);
TRACE_VAL_F(regs->cc, "%lx");
while (!(regs->csts & 1))
while (!NVME_REGS_RDY(regs))
;
TRACE_VAL_F(regs->cc, "%lx");