diff --git a/src/kernel/kernel.c b/src/kernel/kernel.c index 7dd32dd..29fbf25 100644 --- a/src/kernel/kernel.c +++ b/src/kernel/kernel.c @@ -6,6 +6,7 @@ #include "src/kernel/nvme.h" #include "src/kernel/panic.h" #include "src/kernel/path.h" +#include "src/kernel/pci.h" #include "src/kernel/pic.h" #include "src/kernel/pipe.h" #include "src/kernel/process.h" @@ -62,6 +63,7 @@ void kernel_main() { gdt_init(); syscall_init(); + pci_init(); nvme_init(); process_t *kernel = memory_allocate(sizeof(process_t)); diff --git a/src/kernel/nvme.c b/src/kernel/nvme.c index 0950338..419ee04 100644 --- a/src/kernel/nvme.c +++ b/src/kernel/nvme.c @@ -8,6 +8,9 @@ #define SECTOR_SIZE 512 +#define NVME_PCI_CLASS 0x01 +#define NVME_PCI_SUBCLASS 0x08 + typedef volatile struct __attribute__((packed)) { uint64_t cap; uint32_t vs; @@ -22,7 +25,24 @@ typedef volatile struct __attribute__((packed)) { uint64_t acq; } nvme_regs_t; +#define NVME_REGS_NVME_CS_SUPPORTED(r) BITS_R(r->cap, 37, 37) +#define NVME_REGS_MQES(r) BITS_R(r->cap, 15, 0) +#define NVME_REGS_IOCQES_R(r) BITS_R(r->cc, 23, 20) +#define NVME_REGS_IOCQES_W(r, v) BITS_W(r->cc, 23, 20, v) +#define NVME_REGS_IOSQES_R(r) BITS_R(r->cc, 19, 16) +#define NVME_REGS_IOSQES_W(r, v) BITS_W(r->cc, 19, 16, v) +#define NVME_REGS_CSS_R(r) BITS_R(r->cc, 6, 4) +#define NVME_REGS_CSS_W(r, v) BITS_W(r->cc, 6, 4, v) +#define NVME_REGS_EN_R(r) BITS_R(r->cc, 0, 0) +#define NVME_REGS_EN_W(r, v) BITS_W(r->cc, 0, 0, v) +#define NVME_REGS_RDY(r) BITS_R(r->csts, 0, 0) +#define NVME_REGS_ACQS_R(r) BITS_R(r->aqa, 27, 16) +#define NVME_REGS_ACQS_W(r, v) BITS_W(r->aqa, 27, 16, v) +#define NVME_REGS_ASQS_R(r) BITS_R(r->aqa, 11, 0) +#define NVME_REGS_ASQS_W(r, v) BITS_W(r->aqa, 11, 0, v) + static nvme_regs_t *regs = (nvme_regs_t *)(KERNEL_VIRTUAL_BASE + MEMORY_SIZE); +static volatile uint32_t *db_regs = (uint32_t *)(KERNEL_VIRTUAL_BASE + MEMORY_SIZE + 0x1000); // TODO Get stride from caps. #define QUEUE_DEPTH 2 @@ -52,12 +72,15 @@ typedef volatile struct __attribute__((packed)) { uint16_t status; } nvme_cqe_t; -uint8_t admin_cq_phase = 1; +#define NVME_CQE_STATUS(e) BITS_R(e->status, 15, 1) +#define NVME_CQE_P(e) BITS_R(e->status, 0, 0) + +static uint8_t admin_cq_phase = 1; static nvme_sqe_t admin_sq[QUEUE_DEPTH] __attribute__((aligned(PAGE_SIZE))); static uint16_t admin_sq_tail = 0; static nvme_cqe_t admin_cq[QUEUE_DEPTH] __attribute__((aligned(PAGE_SIZE))); static uint16_t admin_cq_head = 0; -uint8_t io_cq_phase = 1; +static uint8_t io_cq_phase = 1; static nvme_sqe_t io_sq[QUEUE_DEPTH] __attribute__((aligned(PAGE_SIZE))); static uint16_t io_sq_tail = 0; static nvme_cqe_t io_cq[QUEUE_DEPTH] __attribute__((aligned(PAGE_SIZE))); @@ -78,12 +101,12 @@ static void admin_exec_sync(nvme_sqe_t *cmd) { memory_copy(cmd, sizeof(nvme_sqe_t), &admin_sq[admin_sq_tail]); TRACE_LN("Advancing submission doorbell..."); - *(volatile uint32_t *)((uint8_t *)regs + 0x1000) = admin_sq_tail = (admin_sq_tail + 1) % QUEUE_DEPTH; + db_regs[0] = admin_sq_tail = (admin_sq_tail + 1) % QUEUE_DEPTH; TRACE_VAL_F(admin_sq_tail, "%u"); TRACE_LN("Waiting for completion..."); nvme_cqe_t *cmp = &admin_cq[admin_cq_head]; - while ((cmp->status & 1) != admin_cq_phase) + while (NVME_CQE_P(cmp) != admin_cq_phase) ; TRACE_LN("Received completion..."); @@ -93,10 +116,10 @@ static void admin_exec_sync(nvme_sqe_t *cmd) { TRACE_VAL_F(cmp->sqid, "%hx"); TRACE_VAL_F(cmp->cid, "%hx"); TRACE_VAL_F(cmp->status, "%hx"); - ASSERT((admin_cq[admin_cq_head].status >> 1) == 0, "admin_exec_sync: NVMe command failed."); + ASSERT(NVME_CQE_STATUS(cmp) == 0, "admin_exec_sync: NVMe command failed."); TRACE_LN("Advancing completion doorbell..."); - *(volatile uint32_t *)((uint8_t *)regs + 0x1004) = admin_cq_head = (admin_cq_head + 1) % QUEUE_DEPTH; // TODO Get offset from caps. + db_regs[1] = admin_cq_head = (admin_cq_head + 1) % QUEUE_DEPTH; admin_cq_phase ^= (admin_cq_head == 0); TRACE_LN("Done."); @@ -117,11 +140,11 @@ static void io_exec_sync(nvme_sqe_t *cmd) { memory_copy(cmd, sizeof(nvme_sqe_t), &io_sq[io_sq_tail]); TRACE_LN("Advancing submission doorbell..."); - *(volatile uint32_t *)((uint8_t *)regs + 0x1008) = io_sq_tail = (io_sq_tail + 1) % QUEUE_DEPTH; // TODO Get offset from caps. + db_regs[2] = io_sq_tail = (io_sq_tail + 1) % QUEUE_DEPTH; // TODO Get offset from caps. TRACE_LN("Waiting for completion..."); nvme_cqe_t *cmp = &io_cq[io_cq_head]; - while ((cmp->status & 1) != io_cq_phase) + while (NVME_CQE_P(cmp) != io_cq_phase) ; TRACE_LN("Received completion..."); @@ -132,10 +155,10 @@ static void io_exec_sync(nvme_sqe_t *cmd) { TRACE_VAL_F(cmp->cid, "%hx"); TRACE_VAL_F(cmp->status, "%hx"); - ASSERT((cmp->status >> 1) == 0, "io_exec_sync: NVMe command failed."); + ASSERT(NVME_CQE_STATUS(cmp) == 0, "io_exec_sync: NVMe command failed."); TRACE_LN("Advancing completion doorbell..."); - *(volatile uint32_t *)((uint8_t *)regs + 0x100C) = io_cq_head = (io_cq_head + 1) % QUEUE_DEPTH; // TODO Get offset from caps. + db_regs[3] = io_cq_head = (io_cq_head + 1) % QUEUE_DEPTH; io_cq_phase ^= (io_cq_head == 0); TRACE_LN("Done."); @@ -144,27 +167,24 @@ static void io_exec_sync(nvme_sqe_t *cmd) { void nvme_init() { LOG_LN("Searching for suitable device..."); uint8_t found = 0; - for (uint16_t bus = 0; bus < 256 && !found; bus++) { - for (uint8_t device = 0; device < 32 && !found; device++) { - for (uint8_t function = 0; function < 8 && !found; function++) { - uint32_t id = pci_read((uint8_t)bus, device, function, 0); - if ((id & 0xFFFF) == 0xFFFF) { + for (uint16_t bus = 0; bus < PCI_MAX_BUSES && !found; bus++) { + for (uint8_t device = 0; device < PCI_MAX_DEVICES && !found; device++) { + for (uint8_t function = 0; function < PCI_MAX_FUNCTIONS && !found; function++) { + uint32_t id = pci_read((uint8_t)bus, device, function, PCI_OFFSET_DEVICE_VENDOR); + if (PCI_DEVICE(id) == PCI_DEVICE_EMPTY) { continue; } - uint32_t class = pci_read((uint8_t)bus, device, function, 8); - if ((class & 0xFF000000) >> 24 != 0x01 || (class & 0x00FF0000) >> 16 != 0x08) { + uint32_t class = pci_read((uint8_t)bus, device, function, PCI_OFFSET_CLASS_REVISION); + if (PCI_CLASS(class) != NVME_PCI_CLASS || PCI_CLASS_SUB(class) != NVME_PCI_SUBCLASS) { continue; } - pci_write(0, 4, 0, 0x04, pci_read(0, 4, 0, 0x04) | 0x06); + LOG_LN("Found device, %hx:%hx.", PCI_VENDOR(id), PCI_DEVICE(id)); - uint32_t bar0 = pci_read((uint8_t)bus, device, function, 0x10) & 0xFFFFFFF0; - uint32_t bar1 = pci_read((uint8_t)bus, device, function, 0x14); - uint64_t phys = ((uint64_t)bar1 << 32) | bar0; - - LOG_LN("Mapping device to memory, phys %lx <-> virt %lx...", phys, regs); + uint64_t bar = pci_bar((uint8_t)bus, device, function); + LOG_LN("Mapping device to memory, phys %lx <-> virt %lx...", bar, regs); for (uint8_t i = 0; i < 4; i++) { - memory_page_map((void *)KERNEL_VIRTUAL_PML4, (void *)((uint8_t *)regs + i * PAGE_SIZE), (void *)((uint8_t *)phys + i * PAGE_SIZE), + memory_page_map((void *)KERNEL_VIRTUAL_PML4, (void *)((uint8_t *)regs + i * PAGE_SIZE), (void *)((uint8_t *)bar + i * PAGE_SIZE), PAGE_WRITABLE | PAGE_PCD | PAGE_USER); } @@ -174,27 +194,32 @@ void nvme_init() { } ASSERT(found, "nvme_init: No suitable devices found."); - ASSERT(regs->cap & (1ULL << 37), "nvme_init: NVM command set not supported."); + ASSERT(NVME_REGS_NVME_CS_SUPPORTED(regs), "nvme_init: NVM command set not supported."); + ASSERT(QUEUE_DEPTH <= NVME_REGS_MQES(regs), "nvme_init: Command queues are too big.") LOG_LN("Disabling device..."); regs->cc = 0; TRACE_VAL_F(regs->cc, "%lx"); - while (regs->csts & 1) + while (NVME_REGS_RDY(regs)) ; TRACE_VAL_F(regs->cc, "%lx"); LOG_LN("Configuring device..."); + NVME_REGS_ASQS_W(regs, QUEUE_DEPTH - 1); regs->asq = (uint64_t)(VIRT_TO_PHYS(admin_sq)); + NVME_REGS_ACQS_W(regs, QUEUE_DEPTH - 1); regs->acq = (uint64_t)(VIRT_TO_PHYS(admin_cq)); - regs->aqa = (QUEUE_DEPTH - 1) << 16 | (QUEUE_DEPTH - 1); TRACE_VAL_F(regs->asq, "%lx"); TRACE_VAL_F(regs->acq, "%lx"); TRACE_VAL_F(regs->aqa, "%lx"); LOG_LN("Enabling device..."); - regs->cc = (4 << 20) | (6 << 16) | (0 << 4) | 1; + NVME_REGS_IOCQES_W(regs, 4); + NVME_REGS_IOSQES_W(regs, 6); + NVME_REGS_CSS_W(regs, 0); + NVME_REGS_EN_W(regs, 1); TRACE_VAL_F(regs->cc, "%lx"); - while (!(regs->csts & 1)) + while (!NVME_REGS_RDY(regs)) ; TRACE_VAL_F(regs->cc, "%lx"); diff --git a/src/kernel/pci.c b/src/kernel/pci.c index efd7079..2343e49 100644 --- a/src/kernel/pci.c +++ b/src/kernel/pci.c @@ -1,12 +1,55 @@ #include "src/kernel/pci.h" +#include "src/kernel/log.h" +#include "src/kernel/panic.h" #include "src/kernel/util.h" +#define PCI_PORT_ADDRESS 0xCF8 +#define PCI_PORT_DATA 0xCFC + +void pci_init() { + LOG_LN("Scanning devices..."); + for (uint16_t bus = 0; bus < PCI_MAX_BUSES; bus++) { + for (uint8_t device = 0; device < PCI_MAX_DEVICES; device++) { + for (uint8_t function = 0; function < PCI_MAX_FUNCTIONS; function++) { + uint32_t id = pci_read((uint8_t)bus, device, function, PCI_OFFSET_DEVICE_VENDOR); + if (PCI_DEVICE(id) == PCI_DEVICE_EMPTY) { + continue; + } + uint32_t class = pci_read((uint8_t)bus, device, function, PCI_OFFSET_CLASS_REVISION); + LOG_LN("%hx:%hx class=%hhx subclass=%hhx, iface=%hhx.", PCI_VENDOR(id), PCI_DEVICE(id), PCI_CLASS(class), PCI_CLASS_SUB(class), + PCI_CLASS_IFACE(class)); + } + } + } + LOG_LN("Done."); +} + uint32_t pci_read(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset) { - outl(0xCF8, ((uint32_t)1 << 31) | ((uint32_t)bus << 16) | ((uint32_t)device << 11) | ((uint32_t)function << 8) | (offset & 0xFC)); - return inl(0xCFC); + // ASSERT(bus < PCI_MAX_BUSES, "pci_read: Malformed address."); + ASSERT(device < PCI_MAX_DEVICES, "pci_read: Malformed address."); + ASSERT(function < PCI_MAX_FUNCTIONS, "pci_read: Malformed address."); + outl(PCI_PORT_ADDRESS, + ((uint32_t)1 << 31) | ((uint32_t)bus << 16) | ((uint32_t)device << 11) | ((uint32_t)function << 8) | (offset & 0xFC)); + return inl(PCI_PORT_DATA); } void pci_write(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset, uint32_t value) { - outl(0xCF8, ((uint32_t)1 << 31) | ((uint32_t)bus << 16) | ((uint32_t)device << 11) | ((uint32_t)function << 8) | (offset & 0xFC)); - outl(0xCFC, value); + // ASSERT(bus < PCI_MAX_BUSES, "pci_write: Malformed address."); + ASSERT(device < PCI_MAX_DEVICES, "pci_write: Malformed address."); + ASSERT(function < PCI_MAX_FUNCTIONS, "pci_write: Malformed address."); + outl(PCI_PORT_ADDRESS, + ((uint32_t)1 << 31) | ((uint32_t)bus << 16) | ((uint32_t)device << 11) | ((uint32_t)function << 8) | (offset & 0xFC)); + outl(PCI_PORT_DATA, value); +} + +uint64_t pci_bar(uint8_t bus, uint8_t device, uint8_t function) { + // ASSERT(bus < PCI_MAX_BUSES, "pci_bar: Malformed address."); + ASSERT(device < PCI_MAX_DEVICES, "pci_bar: Malformed address."); + ASSERT(function < PCI_MAX_FUNCTIONS, "pci_bar: Malformed address."); + + pci_write(bus, device, function, PCI_OFFSET_STATUS_COMMAND, + pci_read(bus, device, function, PCI_OFFSET_STATUS_COMMAND) | PCI_COMMAND_MEMORY_SPACE | PCI_COMMAND_BUS_MASTER); + + return ((uint64_t)pci_read(bus, device, function, PCI_OFFSET_BAR_1) << 32) | + (pci_read(bus, device, function, PCI_OFFSET_BAR_0) & 0xFFFFFFF0); } diff --git a/src/kernel/pci.h b/src/kernel/pci.h index 7ddff29..635fa18 100644 --- a/src/kernel/pci.h +++ b/src/kernel/pci.h @@ -1,7 +1,34 @@ #pragma once #include +#include "src/lib/util.h" + +#define PCI_MAX_BUSES 256 +#define PCI_MAX_DEVICES 32 +#define PCI_MAX_FUNCTIONS 8 + +#define PCI_OFFSET_DEVICE_VENDOR 0x00 +#define PCI_OFFSET_STATUS_COMMAND 0x04 +#define PCI_OFFSET_CLASS_REVISION 0x08 +#define PCI_OFFSET_BAR_0 0x10 +#define PCI_OFFSET_BAR_1 0x14 + +#define PCI_VENDOR(i) BITS_R(i, 15, 0) +#define PCI_DEVICE(i) BITS_R(i, 31, 16) + +#define PCI_DEVICE_EMPTY 0xFFFF + +#define PCI_CLASS(c) BITS_R(c, 31, 24) +#define PCI_CLASS_SUB(c) BITS_R(c, 23, 16) +#define PCI_CLASS_IFACE(c) BITS_R(c, 15, 8) + +#define PCI_COMMAND_MEMORY_SPACE 0x02 +#define PCI_COMMAND_BUS_MASTER 0x04 + +void pci_init(); uint32_t pci_read(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset); void pci_write(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset, uint32_t value); + +uint64_t pci_bar(uint8_t bus, uint8_t device, uint8_t function); diff --git a/src/lib/util.h b/src/lib/util.h index cfa4afb..ffb957a 100644 --- a/src/lib/util.h +++ b/src/lib/util.h @@ -4,6 +4,10 @@ #define NUL 0 // TODO Fix VSCode thinking `NULL` conflicts with some other definition. +#define ONES(h, l) ((1ULL << ((h) - (l) + 1)) - 1) +#define BITS_R(s, h, l) (((s) >> (l)) & ONES(h, l)) +#define BITS_W(s, h, l, v) ((s) = (__typeof__(s))(((s) & ~(ONES(h, l) << (l))) | (((v) & ONES(h, l)) << (l)))) + #define STDIN 0 #define STDOUT 1 #define STDERR 2