Add common way to access bit ranges, log all NVMe devices on startup

src/kernel/kernel.c

@@ -6,6 +6,7 @@
 #include "src/kernel/nvme.h"
 #include "src/kernel/panic.h"
 #include "src/kernel/path.h"
+#include "src/kernel/pci.h"
 #include "src/kernel/pic.h"
 #include "src/kernel/pipe.h"
 #include "src/kernel/process.h"
@@ -62,6 +63,7 @@ void kernel_main() {
 
   gdt_init();
   syscall_init();
+  pci_init();
   nvme_init();
 
   process_t *kernel = memory_allocate(sizeof(process_t));

src/kernel/nvme.c

@@ -8,6 +8,9 @@
 
 #define SECTOR_SIZE 512
 
+#define NVME_PCI_CLASS 0x01
+#define NVME_PCI_SUBCLASS 0x08
+
 typedef volatile struct __attribute__((packed)) {
   uint64_t cap;
   uint32_t vs;
@@ -22,7 +25,24 @@ typedef volatile struct __attribute__((packed)) {
   uint64_t acq;
 } nvme_regs_t;
 
+#define NVME_REGS_NVME_CS_SUPPORTED(r) BITS_R(r->cap, 37, 37)
+#define NVME_REGS_MQES(r) BITS_R(r->cap, 15, 0)
+#define NVME_REGS_IOCQES_R(r) BITS_R(r->cc, 23, 20)
+#define NVME_REGS_IOCQES_W(r, v) BITS_W(r->cc, 23, 20, v)
+#define NVME_REGS_IOSQES_R(r) BITS_R(r->cc, 19, 16)
+#define NVME_REGS_IOSQES_W(r, v) BITS_W(r->cc, 19, 16, v)
+#define NVME_REGS_CSS_R(r) BITS_R(r->cc, 6, 4)
+#define NVME_REGS_CSS_W(r, v) BITS_W(r->cc, 6, 4, v)
+#define NVME_REGS_EN_R(r) BITS_R(r->cc, 0, 0)
+#define NVME_REGS_EN_W(r, v) BITS_W(r->cc, 0, 0, v)
+#define NVME_REGS_RDY(r) BITS_R(r->csts, 0, 0)
+#define NVME_REGS_ACQS_R(r) BITS_R(r->aqa, 27, 16)
+#define NVME_REGS_ACQS_W(r, v) BITS_W(r->aqa, 27, 16, v)
+#define NVME_REGS_ASQS_R(r) BITS_R(r->aqa, 11, 0)
+#define NVME_REGS_ASQS_W(r, v) BITS_W(r->aqa, 11, 0, v)
+
 static nvme_regs_t *regs = (nvme_regs_t *)(KERNEL_VIRTUAL_BASE + MEMORY_SIZE);
+static volatile uint32_t *db_regs = (uint32_t *)(KERNEL_VIRTUAL_BASE + MEMORY_SIZE + 0x1000); // TODO Get stride from caps.
 
 #define QUEUE_DEPTH 2
 
@@ -52,12 +72,15 @@ typedef volatile struct __attribute__((packed)) {
   uint16_t status;
 } nvme_cqe_t;
 
-uint8_t admin_cq_phase = 1;
+#define NVME_CQE_STATUS(e) BITS_R(e->status, 15, 1)
+#define NVME_CQE_P(e) BITS_R(e->status, 0, 0)
+
+static uint8_t admin_cq_phase = 1;
 static nvme_sqe_t admin_sq[QUEUE_DEPTH] __attribute__((aligned(PAGE_SIZE)));
 static uint16_t admin_sq_tail = 0;
 static nvme_cqe_t admin_cq[QUEUE_DEPTH] __attribute__((aligned(PAGE_SIZE)));
 static uint16_t admin_cq_head = 0;
-uint8_t io_cq_phase = 1;
+static uint8_t io_cq_phase = 1;
 static nvme_sqe_t io_sq[QUEUE_DEPTH] __attribute__((aligned(PAGE_SIZE)));
 static uint16_t io_sq_tail = 0;
 static nvme_cqe_t io_cq[QUEUE_DEPTH] __attribute__((aligned(PAGE_SIZE)));
@@ -78,12 +101,12 @@ static void admin_exec_sync(nvme_sqe_t *cmd) {
   memory_copy(cmd, sizeof(nvme_sqe_t), &admin_sq[admin_sq_tail]);
 
   TRACE_LN("Advancing submission doorbell...");
-  *(volatile uint32_t *)((uint8_t *)regs + 0x1000) = admin_sq_tail = (admin_sq_tail + 1) % QUEUE_DEPTH;
+  db_regs[0] = admin_sq_tail = (admin_sq_tail + 1) % QUEUE_DEPTH;
   TRACE_VAL_F(admin_sq_tail, "%u");
 
   TRACE_LN("Waiting for completion...");
   nvme_cqe_t *cmp = &admin_cq[admin_cq_head];
-  while ((cmp->status & 1) != admin_cq_phase)
+  while (NVME_CQE_P(cmp) != admin_cq_phase)
     ;
 
   TRACE_LN("Received completion...");
@@ -93,10 +116,10 @@ static void admin_exec_sync(nvme_sqe_t *cmd) {
   TRACE_VAL_F(cmp->sqid, "%hx");
   TRACE_VAL_F(cmp->cid, "%hx");
   TRACE_VAL_F(cmp->status, "%hx");
-  ASSERT((admin_cq[admin_cq_head].status >> 1) == 0, "admin_exec_sync: NVMe command failed.");
+  ASSERT(NVME_CQE_STATUS(cmp) == 0, "admin_exec_sync: NVMe command failed.");
 
   TRACE_LN("Advancing completion doorbell...");
-  *(volatile uint32_t *)((uint8_t *)regs + 0x1004) = admin_cq_head = (admin_cq_head + 1) % QUEUE_DEPTH; // TODO Get offset from caps.
+  db_regs[1] = admin_cq_head = (admin_cq_head + 1) % QUEUE_DEPTH;
   admin_cq_phase ^= (admin_cq_head == 0);
 
   TRACE_LN("Done.");
@@ -117,11 +140,11 @@ static void io_exec_sync(nvme_sqe_t *cmd) {
   memory_copy(cmd, sizeof(nvme_sqe_t), &io_sq[io_sq_tail]);
 
   TRACE_LN("Advancing submission doorbell...");
-  *(volatile uint32_t *)((uint8_t *)regs + 0x1008) = io_sq_tail = (io_sq_tail + 1) % QUEUE_DEPTH; // TODO Get offset from caps.
+  db_regs[2] = io_sq_tail = (io_sq_tail + 1) % QUEUE_DEPTH; // TODO Get offset from caps.
 
   TRACE_LN("Waiting for completion...");
   nvme_cqe_t *cmp = &io_cq[io_cq_head];
-  while ((cmp->status & 1) != io_cq_phase)
+  while (NVME_CQE_P(cmp) != io_cq_phase)
     ;
 
   TRACE_LN("Received completion...");
@@ -132,10 +155,10 @@ static void io_exec_sync(nvme_sqe_t *cmd) {
   TRACE_VAL_F(cmp->cid, "%hx");
   TRACE_VAL_F(cmp->status, "%hx");
 
-  ASSERT((cmp->status >> 1) == 0, "io_exec_sync: NVMe command failed.");
+  ASSERT(NVME_CQE_STATUS(cmp) == 0, "io_exec_sync: NVMe command failed.");
 
   TRACE_LN("Advancing completion doorbell...");
-  *(volatile uint32_t *)((uint8_t *)regs + 0x100C) = io_cq_head = (io_cq_head + 1) % QUEUE_DEPTH; // TODO Get offset from caps.
+  db_regs[3] = io_cq_head = (io_cq_head + 1) % QUEUE_DEPTH;
   io_cq_phase ^= (io_cq_head == 0);
 
   TRACE_LN("Done.");
@@ -144,27 +167,24 @@ static void io_exec_sync(nvme_sqe_t *cmd) {
 void nvme_init() {
   LOG_LN("Searching for suitable device...");
   uint8_t found = 0;
-  for (uint16_t bus = 0; bus < 256 && !found; bus++) {
+  for (uint16_t bus = 0; bus < PCI_MAX_BUSES && !found; bus++) {
-    for (uint8_t device = 0; device < 32 && !found; device++) {
+    for (uint8_t device = 0; device < PCI_MAX_DEVICES && !found; device++) {
-      for (uint8_t function = 0; function < 8 && !found; function++) {
+      for (uint8_t function = 0; function < PCI_MAX_FUNCTIONS && !found; function++) {
-        uint32_t id = pci_read((uint8_t)bus, device, function, 0);
+        uint32_t id = pci_read((uint8_t)bus, device, function, PCI_OFFSET_DEVICE_VENDOR);
-        if ((id & 0xFFFF) == 0xFFFF) {
+        if (PCI_DEVICE(id) == PCI_DEVICE_EMPTY) {
           continue;
         }
-        uint32_t class = pci_read((uint8_t)bus, device, function, 8);
+        uint32_t class = pci_read((uint8_t)bus, device, function, PCI_OFFSET_CLASS_REVISION);
-        if ((class & 0xFF000000) >> 24 != 0x01 || (class & 0x00FF0000) >> 16 != 0x08) {
+        if (PCI_CLASS(class) != NVME_PCI_CLASS || PCI_CLASS_SUB(class) != NVME_PCI_SUBCLASS) {
           continue;
         }
 
-        pci_write(0, 4, 0, 0x04, pci_read(0, 4, 0, 0x04) | 0x06);
+        LOG_LN("Found device, %hx:%hx.", PCI_VENDOR(id), PCI_DEVICE(id));
 
-        uint32_t bar0 = pci_read((uint8_t)bus, device, function, 0x10) & 0xFFFFFFF0;
+        uint64_t bar = pci_bar((uint8_t)bus, device, function);
-        uint32_t bar1 = pci_read((uint8_t)bus, device, function, 0x14);
+        LOG_LN("Mapping device to memory, phys %lx <-> virt %lx...", bar, regs);
-        uint64_t phys = ((uint64_t)bar1 << 32) | bar0;
-
-        LOG_LN("Mapping device to memory, phys %lx <-> virt %lx...", phys, regs);
         for (uint8_t i = 0; i < 4; i++) {
-          memory_page_map((void *)KERNEL_VIRTUAL_PML4, (void *)((uint8_t *)regs + i * PAGE_SIZE), (void *)((uint8_t *)phys + i * PAGE_SIZE),
+          memory_page_map((void *)KERNEL_VIRTUAL_PML4, (void *)((uint8_t *)regs + i * PAGE_SIZE), (void *)((uint8_t *)bar + i * PAGE_SIZE),
                           PAGE_WRITABLE | PAGE_PCD | PAGE_USER);
         }
 
@@ -174,27 +194,32 @@ void nvme_init() {
   }
 
   ASSERT(found, "nvme_init: No suitable devices found.");
-  ASSERT(regs->cap & (1ULL << 37), "nvme_init: NVM command set not supported.");
+  ASSERT(NVME_REGS_NVME_CS_SUPPORTED(regs), "nvme_init: NVM command set not supported.");
+  ASSERT(QUEUE_DEPTH <= NVME_REGS_MQES(regs), "nvme_init: Command queues are too big.")
 
   LOG_LN("Disabling device...");
   regs->cc = 0;
   TRACE_VAL_F(regs->cc, "%lx");
-  while (regs->csts & 1)
+  while (NVME_REGS_RDY(regs))
     ;
   TRACE_VAL_F(regs->cc, "%lx");
 
   LOG_LN("Configuring device...");
+  NVME_REGS_ASQS_W(regs, QUEUE_DEPTH - 1);
   regs->asq = (uint64_t)(VIRT_TO_PHYS(admin_sq));
+  NVME_REGS_ACQS_W(regs, QUEUE_DEPTH - 1);
   regs->acq = (uint64_t)(VIRT_TO_PHYS(admin_cq));
-  regs->aqa = (QUEUE_DEPTH - 1) << 16 | (QUEUE_DEPTH - 1);
   TRACE_VAL_F(regs->asq, "%lx");
   TRACE_VAL_F(regs->acq, "%lx");
   TRACE_VAL_F(regs->aqa, "%lx");
 
   LOG_LN("Enabling device...");
-  regs->cc = (4 << 20) | (6 << 16) | (0 << 4) | 1;
+  NVME_REGS_IOCQES_W(regs, 4);
+  NVME_REGS_IOSQES_W(regs, 6);
+  NVME_REGS_CSS_W(regs, 0);
+  NVME_REGS_EN_W(regs, 1);
   TRACE_VAL_F(regs->cc, "%lx");
-  while (!(regs->csts & 1))
+  while (!NVME_REGS_RDY(regs))
     ;
   TRACE_VAL_F(regs->cc, "%lx");
 

src/kernel/pci.c

@@ -1,12 +1,55 @@
 #include "src/kernel/pci.h"
+#include "src/kernel/log.h"
+#include "src/kernel/panic.h"
 #include "src/kernel/util.h"
 
+#define PCI_PORT_ADDRESS 0xCF8
+#define PCI_PORT_DATA 0xCFC
+
+void pci_init() {
+  LOG_LN("Scanning devices...");
+  for (uint16_t bus = 0; bus < PCI_MAX_BUSES; bus++) {
+    for (uint8_t device = 0; device < PCI_MAX_DEVICES; device++) {
+      for (uint8_t function = 0; function < PCI_MAX_FUNCTIONS; function++) {
+        uint32_t id = pci_read((uint8_t)bus, device, function, PCI_OFFSET_DEVICE_VENDOR);
+        if (PCI_DEVICE(id) == PCI_DEVICE_EMPTY) {
+          continue;
+        }
+        uint32_t class = pci_read((uint8_t)bus, device, function, PCI_OFFSET_CLASS_REVISION);
+        LOG_LN("%hx:%hx class=%hhx subclass=%hhx, iface=%hhx.", PCI_VENDOR(id), PCI_DEVICE(id), PCI_CLASS(class), PCI_CLASS_SUB(class),
+               PCI_CLASS_IFACE(class));
+      }
+    }
+  }
+  LOG_LN("Done.");
+}
+
 uint32_t pci_read(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset) {
-  outl(0xCF8, ((uint32_t)1 << 31) | ((uint32_t)bus << 16) | ((uint32_t)device << 11) | ((uint32_t)function << 8) | (offset & 0xFC));
+  // ASSERT(bus < PCI_MAX_BUSES, "pci_read: Malformed address.");
-  return inl(0xCFC);
+  ASSERT(device < PCI_MAX_DEVICES, "pci_read: Malformed address.");
+  ASSERT(function < PCI_MAX_FUNCTIONS, "pci_read: Malformed address.");
+  outl(PCI_PORT_ADDRESS,
+       ((uint32_t)1 << 31) | ((uint32_t)bus << 16) | ((uint32_t)device << 11) | ((uint32_t)function << 8) | (offset & 0xFC));
+  return inl(PCI_PORT_DATA);
 }
 
 void pci_write(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset, uint32_t value) {
-  outl(0xCF8, ((uint32_t)1 << 31) | ((uint32_t)bus << 16) | ((uint32_t)device << 11) | ((uint32_t)function << 8) | (offset & 0xFC));
+  // ASSERT(bus < PCI_MAX_BUSES, "pci_write: Malformed address.");
-  outl(0xCFC, value);
+  ASSERT(device < PCI_MAX_DEVICES, "pci_write: Malformed address.");
+  ASSERT(function < PCI_MAX_FUNCTIONS, "pci_write: Malformed address.");
+  outl(PCI_PORT_ADDRESS,
+       ((uint32_t)1 << 31) | ((uint32_t)bus << 16) | ((uint32_t)device << 11) | ((uint32_t)function << 8) | (offset & 0xFC));
+  outl(PCI_PORT_DATA, value);
+}
+
+uint64_t pci_bar(uint8_t bus, uint8_t device, uint8_t function) {
+  // ASSERT(bus < PCI_MAX_BUSES, "pci_bar: Malformed address.");
+  ASSERT(device < PCI_MAX_DEVICES, "pci_bar: Malformed address.");
+  ASSERT(function < PCI_MAX_FUNCTIONS, "pci_bar: Malformed address.");
+
+  pci_write(bus, device, function, PCI_OFFSET_STATUS_COMMAND,
+            pci_read(bus, device, function, PCI_OFFSET_STATUS_COMMAND) | PCI_COMMAND_MEMORY_SPACE | PCI_COMMAND_BUS_MASTER);
+
+  return ((uint64_t)pci_read(bus, device, function, PCI_OFFSET_BAR_1) << 32) |
+         (pci_read(bus, device, function, PCI_OFFSET_BAR_0) & 0xFFFFFFF0);
 }

src/kernel/pci.h

@@ -1,7 +1,34 @@
 #pragma once
 
 #include <stdint.h>
+#include "src/lib/util.h"
+
+#define PCI_MAX_BUSES 256
+#define PCI_MAX_DEVICES 32
+#define PCI_MAX_FUNCTIONS 8
+
+#define PCI_OFFSET_DEVICE_VENDOR 0x00
+#define PCI_OFFSET_STATUS_COMMAND 0x04
+#define PCI_OFFSET_CLASS_REVISION 0x08
+#define PCI_OFFSET_BAR_0 0x10
+#define PCI_OFFSET_BAR_1 0x14
+
+#define PCI_VENDOR(i) BITS_R(i, 15, 0)
+#define PCI_DEVICE(i) BITS_R(i, 31, 16)
+
+#define PCI_DEVICE_EMPTY 0xFFFF
+
+#define PCI_CLASS(c) BITS_R(c, 31, 24)
+#define PCI_CLASS_SUB(c) BITS_R(c, 23, 16)
+#define PCI_CLASS_IFACE(c) BITS_R(c, 15, 8)
+
+#define PCI_COMMAND_MEMORY_SPACE 0x02
+#define PCI_COMMAND_BUS_MASTER 0x04
+
+void pci_init();
 
 uint32_t pci_read(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset);
 
 void pci_write(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset, uint32_t value);
+
+uint64_t pci_bar(uint8_t bus, uint8_t device, uint8_t function);

src/lib/util.h

@@ -4,6 +4,10 @@
 
 #define NUL 0 // TODO Fix VSCode thinking `NULL` conflicts with some other definition.
 
+#define ONES(h, l) ((1ULL << ((h) - (l) + 1)) - 1)
+#define BITS_R(s, h, l) (((s) >> (l)) & ONES(h, l))
+#define BITS_W(s, h, l, v) ((s) = (__typeof__(s))(((s) & ~(ONES(h, l) << (l))) | (((v) & ONES(h, l)) << (l))))
+
 #define STDIN 0
 #define STDOUT 1
 #define STDERR 2