Add simple NVMe module and switch to NVMe emulation

meson.build

@@ -74,9 +74,12 @@ kernel_sources = files([
   'src/kernel/idt.c',
   'src/kernel/kernel.c',
   'src/kernel/keyboard.c',
+  'src/kernel/log.c',
   'src/kernel/memory.c',
+  'src/kernel/nvme.c',
   'src/kernel/panic.c',
   'src/kernel/path.c',
+  'src/kernel/pci.c',
   'src/kernel/pic.c',
   'src/kernel/pipe.c',
   'src/kernel/process.c',

src/kernel/fat16.c

@@ -1,6 +1,6 @@
 #include "src/kernel/fat16.h"
-#include "src/kernel/ata.h"
 #include "src/kernel/fs.h"
+#include "src/kernel/nvme.h"
 #include "src/kernel/panic.h"
 #include "src/kernel/stream.h"
 #include "src/lib/memory.h"
@@ -135,7 +135,7 @@ static void ensure_bpb() {
   }
 
   uint8_t sector[512];
-  ata_read_sectors(FIRST_PARTITION_SECTOR, 1, &sector);
+  nvme_read_sectors(FIRST_PARTITION_SECTOR, 1, &sector);
   bpb = memory_allocate(sizeof(fat16_bpb_t));
   memory_copy(sector + 11, sizeof(fat16_bpb_t), bpb);
   data_start = FIRST_PARTITION_SECTOR + bpb->reserved_sectors + bpb->sectors_per_fat * bpb->fats_count +
@@ -152,7 +152,7 @@ static void ensure_fat() {
   ASSERT(bpb->sectors_per_fat < 256, "ensure_fat: big FAT not implemented");
 
   fat = memory_allocate(bpb->sectors_per_fat * SECTOR_SIZE);
-  ata_read_sectors(FIRST_PARTITION_SECTOR + bpb->reserved_sectors, (uint8_t)bpb->sectors_per_fat, fat);
+  nvme_read_sectors(FIRST_PARTITION_SECTOR + bpb->reserved_sectors, (uint8_t)bpb->sectors_per_fat, fat);
 }
 
 static uint16_t allocate_cluster() {
@@ -168,7 +168,7 @@ static uint16_t allocate_cluster() {
 
 static void flush_fat() {
   ensure_fat();
-  ata_write_sectors(FIRST_PARTITION_SECTOR + bpb->reserved_sectors, (uint8_t)bpb->sectors_per_fat, fat);
+  nvme_write_sectors(FIRST_PARTITION_SECTOR + bpb->reserved_sectors, (uint8_t)bpb->sectors_per_fat, fat);
 }
 
 static uint16_t read_directory(uint16_t dir_cluster, fat16_dir_entry_t **entries) {
@@ -177,7 +177,7 @@ static uint16_t read_directory(uint16_t dir_cluster, fat16_dir_entry_t **entries
   if (!dir_cluster) {
     uint8_t sectors = (uint8_t)((sizeof(fat16_dir_entry_t) * bpb->root_entry_count + SECTOR_SIZE - 1) / SECTOR_SIZE);
     *entries = memory_allocate(sectors * SECTOR_SIZE);
-    ata_read_sectors(FIRST_PARTITION_SECTOR + bpb->reserved_sectors + bpb->fats_count * bpb->sectors_per_fat, sectors, *entries);
+    nvme_read_sectors(FIRST_PARTITION_SECTOR + bpb->reserved_sectors + bpb->fats_count * bpb->sectors_per_fat, sectors, *entries);
     return bpb->root_entry_count;
   } else {
     ensure_fat();
@@ -193,7 +193,7 @@ static uint16_t read_directory(uint16_t dir_cluster, fat16_dir_entry_t **entries
     uint8_t *chunk = (uint8_t *)*entries;
     next_cluster = dir_cluster;
     while (next_cluster < 0xFFF8) {
-      ata_read_sectors(data_start + bpb->sectors_per_cluster * (next_cluster - 2), bpb->sectors_per_cluster, chunk);
+      nvme_read_sectors(data_start + bpb->sectors_per_cluster * (next_cluster - 2), bpb->sectors_per_cluster, chunk);
       chunk += bpb->sectors_per_cluster * SECTOR_SIZE;
       next_cluster = fat[next_cluster];
     }
@@ -223,15 +223,15 @@ static void write_directory(uint16_t dir_cluster, const fat16_dir_entry_t *entri
          "write_directory: entries must be aligned to clusters");
 
   if (!dir_cluster) {
-    ata_write_sectors(FIRST_PARTITION_SECTOR + bpb->reserved_sectors + bpb->fats_count * bpb->sectors_per_fat,
-                      (uint8_t)((sizeof(fat16_dir_entry_t) * bpb->root_entry_count + SECTOR_SIZE - 1) / SECTOR_SIZE), entries);
+    nvme_write_sectors(FIRST_PARTITION_SECTOR + bpb->reserved_sectors + bpb->fats_count * bpb->sectors_per_fat,
+                       (uint8_t)((sizeof(fat16_dir_entry_t) * bpb->root_entry_count + SECTOR_SIZE - 1) / SECTOR_SIZE), entries);
   } else {
     ensure_fat();
 
     uint8_t *chunk = (uint8_t *)entries;
     uint16_t next_cluster = dir_cluster;
     while (count) {
-      ata_write_sectors(data_start + bpb->sectors_per_cluster * (next_cluster - 2), bpb->sectors_per_cluster, chunk);
+      nvme_write_sectors(data_start + bpb->sectors_per_cluster * (next_cluster - 2), bpb->sectors_per_cluster, chunk);
       chunk += bpb->sectors_per_cluster * SECTOR_SIZE;
       count -= bpb->sectors_per_cluster * SECTOR_SIZE / sizeof(fat16_dir_entry_t);
       if (count && fat[next_cluster] >= 0xFFF8) {
@@ -418,7 +418,7 @@ uint64_t fat16_read(const fs_node_t *file, uint32_t offset, uint32_t bytes, void
   uint8_t *cursor = to;
 
   uint8_t *tmp = memory_allocate(cluster_size);
-  ata_read_sectors(data_start + bpb->sectors_per_cluster * (next_cluster - 2), bpb->sectors_per_cluster, tmp);
+  nvme_read_sectors(data_start + bpb->sectors_per_cluster * (next_cluster - 2), bpb->sectors_per_cluster, tmp);
   uint32_t prefix_size = bytes <= cluster_size - offset ? bytes : cluster_size - offset;
   memory_copy(tmp + offset, prefix_size, cursor);
   bytes -= prefix_size;
@@ -427,7 +427,7 @@ uint64_t fat16_read(const fs_node_t *file, uint32_t offset, uint32_t bytes, void
   next_cluster = fat[next_cluster];
 
   while (bytes >= cluster_size) {
-    ata_read_sectors(data_start + bpb->sectors_per_cluster * (next_cluster - 2), bpb->sectors_per_cluster, cursor);
+    nvme_read_sectors(data_start + bpb->sectors_per_cluster * (next_cluster - 2), bpb->sectors_per_cluster, cursor);
     bytes -= cluster_size;
     readden += cluster_size;
     cursor += cluster_size;
@@ -435,7 +435,7 @@ uint64_t fat16_read(const fs_node_t *file, uint32_t offset, uint32_t bytes, void
   }
 
   if (bytes) {
-    ata_read_sectors(data_start + bpb->sectors_per_cluster * (next_cluster - 2), bpb->sectors_per_cluster, tmp);
+    nvme_read_sectors(data_start + bpb->sectors_per_cluster * (next_cluster - 2), bpb->sectors_per_cluster, tmp);
     memory_copy(tmp, bytes, cursor);
     readden += bytes;
   }
@@ -489,17 +489,17 @@ uint64_t fat16_write(fs_node_t *file, uint32_t offset, const void *from, uint32_
   const uint8_t *cursor = from;
 
   uint8_t *tmp = memory_allocate(cluster_size);
-  ata_read_sectors(data_start + bpb->sectors_per_cluster * (next_cluster - 2), bpb->sectors_per_cluster, tmp);
+  nvme_read_sectors(data_start + bpb->sectors_per_cluster * (next_cluster - 2), bpb->sectors_per_cluster, tmp);
   uint32_t prefix_size = bytes <= cluster_size - offset ? bytes : cluster_size - offset;
   memory_copy(cursor, prefix_size, tmp + offset);
-  ata_write_sectors(data_start + bpb->sectors_per_cluster * (next_cluster - 2), bpb->sectors_per_cluster, tmp);
+  nvme_write_sectors(data_start + bpb->sectors_per_cluster * (next_cluster - 2), bpb->sectors_per_cluster, tmp);
   bytes -= prefix_size;
   written += prefix_size;
   cursor += prefix_size;
   next_cluster = fat[next_cluster];
 
   while (bytes >= cluster_size) {
-    ata_write_sectors(data_start + bpb->sectors_per_cluster * (next_cluster - 2), bpb->sectors_per_cluster, cursor);
+    nvme_write_sectors(data_start + bpb->sectors_per_cluster * (next_cluster - 2), bpb->sectors_per_cluster, cursor);
     bytes -= cluster_size;
     written += cluster_size;
     cursor += cluster_size;
@@ -507,9 +507,9 @@ uint64_t fat16_write(fs_node_t *file, uint32_t offset, const void *from, uint32_
   }
 
   if (bytes) {
-    ata_read_sectors(data_start + bpb->sectors_per_cluster * (next_cluster - 2), bpb->sectors_per_cluster, tmp);
+    nvme_read_sectors(data_start + bpb->sectors_per_cluster * (next_cluster - 2), bpb->sectors_per_cluster, tmp);
     memory_copy(cursor, bytes, tmp);
-    ata_write_sectors(data_start + bpb->sectors_per_cluster * (next_cluster - 2), bpb->sectors_per_cluster, tmp);
+    nvme_write_sectors(data_start + bpb->sectors_per_cluster * (next_cluster - 2), bpb->sectors_per_cluster, tmp);
     written += bytes;
   }
 

src/kernel/idt.c

@@ -38,3 +38,7 @@ void idt_set_entry(int vector, void (*handler)(struct interrupt_frame *), uint8_
   idt[vector].offset_high = (addr >> 32) & 0xFFFFFFFF;
   idt[vector].zero = 0;
 }
+
+void idt_set_entry_ec(int vector, void (*handler)(struct interrupt_frame *, uint64_t error), uint8_t flags) {
+  idt_set_entry(vector, (void *)handler, flags);
+}

src/kernel/idt.h

@@ -13,3 +13,5 @@ struct interrupt_frame {
 void idt_init();
 
 void idt_set_entry(int vector, void (*handler)(struct interrupt_frame *), uint8_t flags);
+
+void idt_set_entry_ec(int vector, void (*handler)(struct interrupt_frame *, uint64_t error), uint8_t flags);

src/kernel/kernel.c

@@ -2,11 +2,14 @@
 #include "src/kernel/gdt.h"
 #include "src/kernel/idt.h"
 #include "src/kernel/keyboard.h"
+#include "src/kernel/log.h"
+#include "src/kernel/nvme.h"
 #include "src/kernel/panic.h"
 #include "src/kernel/path.h"
 #include "src/kernel/pic.h"
 #include "src/kernel/pipe.h"
 #include "src/kernel/process.h"
+#include "src/kernel/stream.h"
 #include "src/kernel/syscall.h"
 #include "src/kernel/timer.h"
 #include "src/kernel/tss.h"
@@ -22,8 +25,14 @@ __attribute__((interrupt)) void isr_divide_by_zero(__attribute__((unused)) struc
     ;
 }
 
-__attribute__((interrupt)) void isr_page_fault(__attribute__((unused)) struct interrupt_frame *frame) {
-  vga_set_string(VGA_HEIGHT - 1, 0, "EXCEPTION: page fault", 0x4F);
+__attribute__((interrupt)) void isr_page_fault(__attribute__((unused)) struct interrupt_frame *frame, uint64_t error_code) {
+  uint64_t cr2;
+  __asm__ volatile("mov %%cr2, %0" : "=r"(cr2));
+
+  char msg[80];
+  string_format("EXCEPTION: page fault; addr=%x err=%x", 80, msg, cr2, error_code);
+
+  vga_set_string(VGA_HEIGHT - 1, 0, msg, 0x4F);
   while (1)
     ;
 }
@@ -40,16 +49,20 @@ __attribute__((interrupt)) void isr_ata_primary(__attribute__((unused)) struct i
 }
 
 void kernel_main() {
+  stream_t *vga = vga_init();
+  log_init(vga);
+
   pic_init();
   idt_init();
   outb(0x21, inb(0x21) | 0x01); // mask out timer interrupt
   idt_set_entry(0, isr_divide_by_zero, 0x8E);
-  idt_set_entry(0x0E, isr_page_fault, 0x8E);
+  idt_set_entry_ec(0x0E, isr_page_fault, 0x8E);
   idt_set_entry(0x0D, isr_general_violation, 0x8E);
   idt_set_entry(46, isr_ata_primary, 0x8E);
 
   gdt_init();
   syscall_init();
+  nvme_init();
 
   process_t *kernel = memory_allocate(sizeof(process_t));
   kernel->pml4 = (uint64_t *)KERNEL_VIRTUAL_PML4;
@@ -59,8 +72,7 @@ void kernel_main() {
   kernel->user_stack = kernel->user_rsp = (uint8_t *)memory_allocate(PAGE_SIZE) + PAGE_SIZE;
   kernel->cwd = memory_allocate(sizeof(path_t));
   kernel->fds[STDIN] = keyboard_init();
-  kernel->fds[STDOUT] = vga_init();
-  kernel->fds[STDERR] = kernel->fds[STDOUT];
+  kernel->fds[STDOUT] = kernel->fds[STDERR] = vga;
   kernel->free_fd = STDERR + 1;
   kernel->code = EXIT_CODE_OK;
 

src/kernel/log.c

@@ -0,0 +1,8 @@
+#include "src/kernel/log.h"
+#include "src/kernel/stream.h"
+
+stream_t *kernel_log;
+
+void log_init(stream_t *log) {
+  kernel_log = log;
+}

src/kernel/log.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#include "src/kernel/stream.h"
+#include "src/lib/memory.h"
+#include "src/lib/string.h"
+
+extern stream_t *kernel_log;
+
+#define LOG(fmt, ...)                                                                                                                      \
+  do {                                                                                                                                     \
+    uint64_t _size = string_length(fmt) * 2;                                                                                               \
+    char *_string = memory_allocate(_size);                                                                                                \
+    _size = string_format(fmt, _size, _string, ##__VA_ARGS__);                                                                             \
+    kernel_log->write(kernel_log, _string, _size);                                                                                         \
+    memory_free(_string);                                                                                                                  \
+  } while (0)
+
+#define TRACE(fmt, ...)
+
+void log_init(stream_t *log);

src/kernel/memory.c

@@ -1,4 +1,5 @@
 #include "src/kernel/memory.h"
+#include "src/kernel/log.h"
 #include "src/kernel/panic.h"
 #include "src/lib/layout.h"
 #include "src/lib/memory.h"
@@ -66,7 +67,10 @@ void memory_page_map(uint64_t *pml4, void *virt, void *phys, uint64_t flags) {
   const uint64_t pd_index = (ivirt >> 21) & 0x1FF;
   const uint64_t pt_index = (ivirt >> 12) & 0x1FF;
 
+  TRACE("memory_page_map: Mapping physical %x to virtual %x / [%u, %u, %u, %u]...\n", phys, virt, pml4_index, pdpt_index, pd_index, pt_index);
+
   if (!(pml4[pml4_index] & PAGE_PRESENT)) {
+    TRACE("memory_page_map: PML4 entry %u does not exist, creating...\n", pml4_index);
     uint64_t *pdpt = memory_page_allocate();
     memory_set(0, PAGE_SIZE, PHYS_TO_VIRT(pdpt));
     pml4[pml4_index] = (uint64_t)pdpt | PAGE_PRESENT | PAGE_WRITABLE | PAGE_USER;
@@ -74,21 +78,32 @@ void memory_page_map(uint64_t *pml4, void *virt, void *phys, uint64_t flags) {
 
   uint64_t *pdpt = PHYS_TO_VIRT(pml4[pml4_index] & ~(uint64_t)0xFFF);
   if (!(pdpt[pdpt_index] & PAGE_PRESENT)) {
+    TRACE("memory_page_map: PDPT entry %u does not exist, creating...\n", pdpt_index);
     uint64_t *pd = memory_page_allocate();
     memory_set(0, PAGE_SIZE, PHYS_TO_VIRT(pd));
     pdpt[pdpt_index] = (uint64_t)pd | PAGE_PRESENT | PAGE_WRITABLE | PAGE_USER;
   }
 
   uint64_t *pd = PHYS_TO_VIRT(pdpt[pdpt_index] & ~(uint64_t)0xFFF);
-  ASSERT(!(pd[pd_index] & 0x80), "memory_page_map: huge page in PD");
+  ASSERT(!(pd[pd_index] & 0x80), "memory_page_map: Huge page in PD.");
   if (!(pd[pd_index] & PAGE_PRESENT)) {
+    TRACE("memory_page_map: PD entry %u does not exist, creating...\n", pd_index);
     uint64_t *pt = memory_page_allocate();
     memory_set(0, PAGE_SIZE, PHYS_TO_VIRT(pt));
     pd[pd_index] = (uint64_t)pt | PAGE_PRESENT | PAGE_WRITABLE | PAGE_USER;
   }
 
   uint64_t *pt = PHYS_TO_VIRT(pd[pd_index] & ~(uint64_t)0xFFF);
+
+  ASSERT(!(pt[pt_index] & PAGE_PRESENT), "memory_page_map: Page already mapped.")
+
+  TRACE("memory_page_map: PT entry %u does not exist, creating...\n", pt_index);
+
   pt[pt_index] = (uint64_t)phys | flags | PAGE_PRESENT;
+
+  TRACE("memory_page_map: Created %x.\n", pt[pt_index]);
+
+  TRACE("memory_page_map: Done.\n");
 }
 
 void memory_page_unmap(uint64_t *pml4, void *virt) {

src/kernel/nvme.c

@@ -0,0 +1,281 @@
+#include "src/kernel/nvme.h"
+#include "src/kernel/log.h"
+#include "src/kernel/memory.h"
+#include "src/kernel/panic.h"
+#include "src/kernel/pci.h"
+#include "src/lib/layout.h"
+#include "src/lib/memory.h"
+#include <stdint.h>
+
+#define SECTOR_SIZE 512
+
+typedef volatile struct __attribute__((packed)) {
+  uint64_t cap;
+  uint32_t vs;
+  uint32_t intms;
+  uint32_t intmc;
+  uint32_t cc;
+  uint32_t rsvd;
+  uint32_t csts;
+  uint32_t nssr;
+  uint32_t aqa;
+  uint64_t asq;
+  uint64_t acq;
+} nvme_regs_t;
+
+static nvme_regs_t *regs = (nvme_regs_t *)(KERNEL_VIRTUAL_BASE + MEMORY_SIZE);
+
+#define QUEUE_DEPTH 2
+
+typedef volatile struct __attribute__((packed)) {
+  uint8_t opc;
+  uint8_t flags;
+  uint16_t cid;
+  uint32_t nsid;
+  uint64_t reserved;
+  uint64_t mptr;
+  uint64_t prp1;
+  uint64_t prp2;
+  uint32_t cdw10;
+  uint32_t cdw11;
+  uint32_t cdw12;
+  uint32_t cdw13;
+  uint32_t cdw14;
+  uint32_t cdw15;
+} nvme_sqe_t;
+
+typedef volatile struct __attribute__((packed)) {
+  uint32_t dw0;
+  uint32_t reserved;
+  uint16_t sqhd;
+  uint16_t sqid;
+  uint16_t cid;
+  uint16_t status;
+} nvme_cqe_t;
+
+uint8_t admin_cq_phase = 1;
+static nvme_sqe_t admin_sq[QUEUE_DEPTH] __attribute__((aligned(PAGE_SIZE)));
+static uint16_t admin_sq_tail = 0;
+static nvme_cqe_t admin_cq[QUEUE_DEPTH] __attribute__((aligned(PAGE_SIZE)));
+static uint16_t admin_cq_head = 0;
+uint8_t io_cq_phase = 1;
+static nvme_sqe_t io_sq[QUEUE_DEPTH] __attribute__((aligned(PAGE_SIZE)));
+static uint16_t io_sq_tail = 0;
+static nvme_cqe_t io_cq[QUEUE_DEPTH] __attribute__((aligned(PAGE_SIZE)));
+static uint16_t io_cq_head = 0;
+
+static void admin_exec_sync(nvme_sqe_t *cmd) {
+  cmd->cid = admin_sq_tail;
+
+  TRACE("admin_exec_sync: Queueing submission...\n");
+  TRACE("admin_exec_sync: opc=%x\n", cmd->opc);
+  TRACE("admin_exec_sync: nsid=%x\n", cmd->nsid);
+  TRACE("admin_exec_sync: cdw10=%x\n", cmd->cdw10);
+  TRACE("admin_exec_sync: cdw11=%x\n", cmd->cdw11);
+  TRACE("admin_exec_sync: cdw12=%x\n", cmd->cdw12);
+  TRACE("admin_exec_sync: prp1=%x\n", cmd->prp1);
+  TRACE("admin_exec_sync: prp2=%x\n", cmd->prp2);
+
+  memory_copy(cmd, sizeof(nvme_sqe_t), &admin_sq[admin_sq_tail]);
+
+  TRACE("admin_exec_sync: Advancing submission doorbell...\n");
+
+  *(volatile uint32_t *)((uint8_t *)regs + 0x1000) = admin_sq_tail = (admin_sq_tail + 1) % QUEUE_DEPTH;
+
+  TRACE("admin_exec_sync: Waiting for completion...\n");
+
+  while ((admin_cq[admin_cq_head].status & 1) != admin_cq_phase)
+    ;
+
+  TRACE("admin_exec_sync: Received completion...\n");
+  TRACE("admin_exec_sync: dw0=%x\n", io_cq[io_cq_head].dw0);
+  TRACE("admin_exec_sync: reserved=%x\n", io_cq[io_cq_head].reserved);
+  TRACE("admin_exec_sync: sqhd=%x\n", io_cq[io_cq_head].sqhd);
+  TRACE("admin_exec_sync: sqid=%x\n", io_cq[io_cq_head].sqid);
+  TRACE("admin_exec_sync: cid=%x\n", io_cq[io_cq_head].cid);
+  TRACE("admin_exec_sync: status=%x\n", io_cq[io_cq_head].status);
+
+  ASSERT((admin_cq[admin_cq_head].status >> 1) == 0, "admin_exec_sync: NVMe command failed.");
+
+  TRACE("admin_exec_sync: Advancing completion doorbell...\n");
+
+  *(volatile uint32_t *)((uint8_t *)regs + 0x1004) = admin_cq_head = (admin_cq_head + 1) % QUEUE_DEPTH; // TODO Get offset from caps.
+
+  admin_cq_phase ^= (admin_cq_head == 0);
+
+  TRACE("admin_exec_sync: Done.\n");
+}
+
+static void io_exec_sync(nvme_sqe_t *cmd) {
+  cmd->cid = io_sq_tail;
+
+  TRACE("io_exec_sync: Queueing submission...\n");
+  TRACE("io_exec_sync: opc=%x\n", cmd->opc);
+  TRACE("io_exec_sync: nsid=%x\n", cmd->nsid);
+  TRACE("io_exec_sync: cdw10=%x\n", cmd->cdw10);
+  TRACE("io_exec_sync: cdw11=%x\n", cmd->cdw11);
+  TRACE("io_exec_sync: cdw12=%x\n", cmd->cdw12);
+  TRACE("io_exec_sync: prp1=%x\n", cmd->prp1);
+  TRACE("io_exec_sync: prp2=%x\n", cmd->prp2);
+
+  memory_copy(cmd, sizeof(nvme_sqe_t), &io_sq[io_sq_tail]);
+
+  TRACE("io_exec_sync: Advancing submission doorbell...\n");
+
+  *(volatile uint32_t *)((uint8_t *)regs + 0x1008) = io_sq_tail = (io_sq_tail + 1) % QUEUE_DEPTH; // TODO Get offset from caps.
+
+  TRACE("io_exec_sync: Waiting for completion...\n");
+
+  while ((io_cq[io_cq_head].status & 1) != io_cq_phase)
+    ;
+
+  TRACE("io_exec_sync: Received completion...\n");
+  TRACE("io_exec_sync: dw0=%x\n", io_cq[io_cq_head].dw0);
+  TRACE("io_exec_sync: reserved=%x\n", io_cq[io_cq_head].reserved);
+  TRACE("io_exec_sync: sqhd=%x\n", io_cq[io_cq_head].sqhd);
+  TRACE("io_exec_sync: sqid=%x\n", io_cq[io_cq_head].sqid);
+  TRACE("io_exec_sync: cid=%x\n", io_cq[io_cq_head].cid);
+  TRACE("io_exec_sync: status=%x\n", io_cq[io_cq_head].status);
+
+  ASSERT((io_cq[io_cq_head].status >> 1) == 0, "io_exec_sync: NVMe command failed.");
+
+  TRACE("io_exec_sync: Advancing completion doorbell...\n");
+
+  *(volatile uint32_t *)((uint8_t *)regs + 0x100C) = io_cq_head = (io_cq_head + 1) % QUEUE_DEPTH; // TODO Get offset from caps.
+
+  io_cq_phase ^= (io_cq_head == 0);
+
+  TRACE("io_exec_sync: Done.\n");
+}
+
+void nvme_init() {
+  LOG("nvme_init: Searching for suitable device...\n");
+  uint8_t found = 0;
+  for (uint16_t bus = 0; bus < 256 && !found; bus++) {
+    for (uint8_t device = 0; device < 32 && !found; device++) {
+      for (uint8_t function = 0; function < 8 && !found; function++) {
+        uint32_t id = pci_read((uint8_t)bus, device, function, 0);
+        if ((id & 0xFFFF) == 0xFFFF) {
+          continue;
+        }
+        uint32_t class = pci_read((uint8_t)bus, device, function, 8);
+        if ((class & 0xFF000000) >> 24 != 0x01 || (class & 0x00FF0000) >> 16 != 0x08) {
+          continue;
+        }
+
+        pci_write(0, 4, 0, 0x04, pci_read(0, 4, 0, 0x04) | 0x06);
+
+        uint32_t bar0 = pci_read((uint8_t)bus, device, function, 0x10) & 0xFFFFFFF0;
+        uint32_t bar1 = pci_read((uint8_t)bus, device, function, 0x14);
+        uint64_t phys = ((uint64_t)bar1 << 32) | bar0;
+
+        LOG("nvme_init: Mapping device to memory, phys %x <-> virt %x\n", phys, regs);
+        for (uint8_t i = 0; i < 4; i++) {
+          memory_page_map((void *)KERNEL_VIRTUAL_PML4, (void *)((uint8_t *)regs + i * PAGE_SIZE), (void *)((uint8_t *)phys + i * PAGE_SIZE),
+                          PAGE_WRITABLE | PAGE_PCD | PAGE_USER);
+        }
+        found = 1;
+      }
+    }
+  }
+
+  ASSERT(found, "nvme_init: No suitable devices found.");
+  ASSERT(regs->cap & (1ULL << 37), "nvme_init: NVM command set not supported.");
+
+  LOG("nvme_init: Disabling device...\n");
+
+  regs->cc = 0;
+  while (regs->csts & 1)
+    ;
+
+  LOG("nvme_init: Configuring device...\n");
+  regs->asq = (uint64_t)(VIRT_TO_PHYS(admin_sq));
+  TRACE("nvme_init: Setting ASQ, expected=%x, set=%x...\n", (uint64_t)(VIRT_TO_PHYS(admin_sq)), regs->asq);
+  regs->acq = (uint64_t)(VIRT_TO_PHYS(admin_cq));
+  TRACE("nvme_init: Setting ACQ, expected=%x, set=%x...\n", (uint64_t)(VIRT_TO_PHYS(admin_cq)), regs->acq);
+  regs->aqa = (QUEUE_DEPTH - 1) << 16 | (QUEUE_DEPTH - 1);
+  TRACE("nvme_init: Setting AQA, expected=%x, set=%x...\n", (QUEUE_DEPTH - 1) << 16 | (QUEUE_DEPTH - 1), regs->aqa);
+
+  regs->cc = (4 << 20) | (6 << 16) | (0 << 4) | 1;
+  LOG("nvme_init: Enabling device, expected=%x, set=%x...\n", (4 << 20) | (6 << 16) | (0 << 4) | 1, regs->cc);
+  while (!(regs->csts & 1))
+    ;
+
+  LOG("nvme_init: Creating I/O completion queue...\n");
+
+  nvme_sqe_t create_io_cq = {
+      .opc = 0x05,
+      .prp1 = (uint64_t)VIRT_TO_PHYS(io_cq),
+      .cdw10 = ((QUEUE_DEPTH - 1) << 16) | 1,
+      .cdw11 = 1,
+  };
+  admin_exec_sync(&create_io_cq);
+
+  LOG("nvme_init: Creating I/O submission queue...\n");
+
+  nvme_sqe_t create_io_sq = {
+      .opc = 0x01,
+      .prp1 = (uint64_t)VIRT_TO_PHYS(io_sq),
+      .cdw10 = ((QUEUE_DEPTH - 1) << 16) | 1,
+      .cdw11 = (1 << 16) | 1,
+  };
+  admin_exec_sync(&create_io_sq);
+
+  LOG("nvme_init: Done.\n");
+}
+
+void nvme_read_sectors(uint32_t index, uint8_t count, void *to) {
+  TRACE("nvme_read_sectors: Reading %u sectors at %u...\n", count, index);
+
+  while (count) {
+    uint64_t remainder = PAGE_SIZE - (uint64_t)to % PAGE_SIZE;
+
+    uint8_t i_count = (uint8_t)((remainder + PAGE_SIZE) / SECTOR_SIZE);
+    i_count = i_count > count ? count : i_count;
+
+    nvme_sqe_t read_sq = {
+        .opc = 0x02,
+        .nsid = 1,
+        .cdw10 = index,
+        .cdw11 = 0,
+        .cdw12 = i_count - 1,
+        .prp1 = (uint64_t)VIRT_TO_PHYS(to),
+        .prp2 = i_count * SECTOR_SIZE > remainder ? (uint64_t)VIRT_TO_PHYS(to + remainder) : 0,
+    };
+    io_exec_sync(&read_sq);
+
+    index += i_count;
+    count -= i_count;
+    to = (void *)((uint8_t *)to + i_count * SECTOR_SIZE);
+  }
+
+  TRACE("nvme_read_sectors: Done.\n", count, index);
+}
+
+void nvme_write_sectors(uint32_t index, uint8_t count, const void *from) {
+  TRACE("nvme_write_sectors: Reading %u sectors at %u...\n", count, index);
+
+  while (count) {
+    uint64_t remainder = PAGE_SIZE - (uint64_t)from % PAGE_SIZE;
+
+    uint8_t i_count = (uint8_t)((remainder + PAGE_SIZE) / SECTOR_SIZE);
+    i_count = i_count > count ? count : i_count;
+
+    nvme_sqe_t write_sq = {
+        .opc = 0x01,
+        .nsid = 1,
+        .cdw10 = index,
+        .cdw11 = 0,
+        .cdw12 = i_count - 1,
+        .prp1 = (uint64_t)VIRT_TO_PHYS(from),
+        .prp2 = i_count * SECTOR_SIZE > remainder ? (uint64_t)VIRT_TO_PHYS(from + remainder) : 0,
+    };
+    io_exec_sync(&write_sq);
+
+    index += i_count;
+    count -= i_count;
+    from = (void *)((uint8_t *)from + i_count * SECTOR_SIZE);
+  }
+
+  TRACE("nvme_write_sectors: Done.\n", count, index);
+}

src/kernel/nvme.h

@@ -0,0 +1,9 @@
+#pragma once
+
+#include <stdint.h>
+
+void nvme_init(); // Assuming one and only one NVMe device.
+
+void nvme_read_sectors(uint32_t index, uint8_t count, void *to);
+
+void nvme_write_sectors(uint32_t index, uint8_t count, const void *from);

src/kernel/pci.c

@@ -0,0 +1,12 @@
+#include "src/kernel/pci.h"
+#include "src/kernel/util.h"
+
+uint32_t pci_read(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset) {
+  outl(0xCF8, ((uint32_t)1 << 31) | ((uint32_t)bus << 16) | ((uint32_t)device << 11) | ((uint32_t)function << 8) | (offset & 0xFC));
+  return inl(0xCFC);
+}
+
+void pci_write(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset, uint32_t value) {
+  outl(0xCF8, ((uint32_t)1 << 31) | ((uint32_t)bus << 16) | ((uint32_t)device << 11) | ((uint32_t)function << 8) | (offset & 0xFC));
+  outl(0xCFC, value);
+}

src/kernel/pci.h

@@ -0,0 +1,7 @@
+#pragma once
+
+#include <stdint.h>
+
+uint32_t pci_read(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset);
+
+void pci_write(uint8_t bus, uint8_t device, uint8_t function, uint8_t offset, uint32_t value);

src/kernel/util.h

@@ -22,6 +22,16 @@ static inline void outw(uint16_t port, uint16_t value) {
   __asm__ volatile("outw %0, %1" : : "a"(value), "Nd"(port));
 }
 
+static inline uint32_t inl(uint16_t port) {
+  uint32_t value;
+  __asm__ volatile("inl %1, %0" : "=a"(value) : "Nd"(port));
+  return value;
+}
+
+static inline void outl(uint16_t port, uint32_t value) {
+  __asm__ volatile("outl %0, %1" : : "a"(value), "Nd"(port));
+}
+
 static inline void insw(uint16_t port, void *to, uint32_t count) {
   __asm__ volatile("rep insw" : "=D"(to), "=c"(count) : "d"(port), "D"(to), "c"(count) : "memory");
 }

src/lib/memory.h

@@ -6,11 +6,11 @@ void *memory_allocate(uint64_t size);
 
 void memory_free(void *pointer);
 
-static inline void memory_set(uint8_t value, uint64_t bytes, void *to) {
+static inline void memory_set(uint8_t value, uint64_t bytes, volatile void *to) {
   __asm__ volatile("rep stosb" : "=D"(to), "=c"(bytes) : "D"(to), "a"(value), "c"(bytes) : "memory");
 }
 
-static inline void memory_move(void *from, uint64_t bytes, void *to) {
+static inline void memory_move(volatile void *from, uint64_t bytes, volatile void *to) {
   if (to == from) {
     return;
   } else if (to < from) {
@@ -20,7 +20,7 @@ static inline void memory_move(void *from, uint64_t bytes, void *to) {
   }
 }
 
-static inline void memory_copy(const void *from, uint64_t bytes, void *to) {
+static inline void memory_copy(volatile const void *from, uint64_t bytes, volatile void *to) {
   // TODO ASSERT((uint64_t)to + bytes <= (uint64_t)from || (uint64_t)to >= (uint64_t)from + bytes, "memory_copy: overlapping backward
   // copy");
 

vm.sh

@@ -9,6 +9,7 @@ fi
 qemu-system-x86_64 \
   "${debug_flags[@]}" \
   -monitor stdio \
-  -drive file="build/os.img",format=raw,if=ide \
+  -drive file=build/os.img,format=raw,if=none,id=nvme0 \
+  -device nvme,drive=nvme0,serial=foo \
   -no-reboot \
   -d int