Kernel: Implement NVMe driver

I'm  actually able to boot this os fine on own laptop now!

bootloader/bios/CMakeLists.txt

@@ -3,6 +3,7 @@ cmake_minimum_required(VERSION 3.26)
 project(bootloader ASM)
 
 set(BOOTLOADER_SOURCES
+	a20_line.S
 	boot.S
 	command_line.S
 	disk.S

bootloader/bios/a20_line.S

@@ -0,0 +1,168 @@
+.code16
+.section .stage2
+
+# checks whether A20 line is enabled or disabled
+# return
+#	ax: 1 if enabled, 0 otherwise
+check_a20:
+	pushf
+	pushw %si
+	pushw %di
+	pushw %ds
+	pushw %es
+
+	cli
+
+	xorw %ax, %ax
+	movw %ax, %es
+	notw %ax
+	movw %ax, %ds
+
+	movw $0x0500, %di
+	movw $0x0510, %si
+
+	movb %es:(%di), %al
+	pushw %ax
+
+	movb %ds:(%si), %al
+	pushw %ax
+
+	movb $0x00, %es:(%di)
+	movb $0xFF, %ds:(%si)
+
+	cmpb $0xFF, %es:(%di)
+	
+	pop %ax
+	movb %al, %ds:(%si)
+
+	pop %ax
+	movb %al, %es:(%di)
+
+	movw $0, %ax
+	je .check_a20_done
+	
+	movw $1, %ax
+
+ .check_a20_done:
+	popw %es
+	popw %ds
+	popw %di
+	popw %si
+	popf
+	ret
+
+
+# Try to enable A20 using PS2 controller
+enable_a20_ps2:
+	pushf
+	pushw %ax
+
+	cli
+
+	# disable first port
+	call .enable_a20_ps2_wait1
+	movb $0xAD, %al
+	outb %al, $0x64
+
+	# read controller output
+	call .enable_a20_ps2_wait1
+	movb $0xD0, %al
+	outb %al, $0x64
+ 
+	call .enable_a20_ps2_wait2
+	inb $0x60, %al
+	pushw %ax
+
+	# write controller output
+	call .enable_a20_ps2_wait1
+	movb $0xD1, %al
+	outb %al, $0x64
+ 
+	call .enable_a20_ps2_wait1
+	popw %ax
+	orw $2, %ax
+	outb %al, $0x60
+
+	# enable first port
+	call .enable_a20_ps2_wait1
+	movb $0xAE, %al
+	outb %al, $0x64
+
+	call .enable_a20_ps2_wait1
+
+	popw %ax
+	popf
+	ret
+
+ .enable_a20_ps2_wait1:
+	inb $0x64, %al
+	test $2, %al
+	jnz .enable_a20_ps2_wait1
+	ret
+
+ .enable_a20_ps2_wait2:
+	inb $0x64, %al
+	test $1, %al
+	jnz .enable_a20_ps2_wait1
+	ret
+
+
+# Check if A20 line is disabled. If it is, try to enable it
+.global enable_a20
+enable_a20:
+	pushw %ax
+	pushw %si
+
+	call check_a20
+	testw %ax, %ax
+	jnz .enable_a20_done
+
+	movw $a20_line_disabled_msg, %si
+	call puts; call print_newline
+
+	# Try to enable A20 line using bios interrupt
+	movw $0x2401, %ax
+	int $0x15
+	call check_a20
+	testw %ax, %ax
+	jnz .enable_a20_done
+
+	# Try to enable A20 line using ps2 controller
+	call enable_a20_ps2
+	call check_a20
+	testw %ax, %ax
+	jnz .enable_a20_done
+
+	# Try to enable A20 line using fast A20 gate
+	inb $0x92, %al
+	testb $2, %al
+	jnz .enable_a20_fast_done
+	orb $2, %al
+	outb %al, $0x92
+ .enable_a20_fast_done:
+
+	call check_a20
+	testw %ax, %ax
+	jnz .enable_a20_done
+
+	movw $a20_could_not_enable_msg, %si
+	call print_and_halt
+
+ .enable_a20_done:
+	movw $a20_line_enabled_msg, %si
+	call puts; call print_newline
+
+	popw %si
+	popw %ax
+	ret
+
+.section .data
+
+a20_line_disabled_msg:
+	.asciz "A20 line disabled. Trying to enable it"
+
+a20_line_enabled_msg:
+	.asciz "A20 line enabled"
+
+a20_could_not_enable_msg:
+	.asciz "Could not enable A20 line"

bootloader/bios/boot.S

@@ -57,7 +57,11 @@ stage2_main:
 	movw $unreal_enter_msg, %si
 	call puts; call print_newline
 
+	call enable_a20
+
 	call get_memory_map
+
+	call print_newline
 	call read_user_command_line
 
 	call print_newline
@@ -129,7 +133,7 @@ enter_unreal_mode:
 	movw $0x10, %bx
 	movw %bx, %ds
 
-	andb 0xFE, %al
+	andb $0xFE, %al
 	movl %eax, %cr0
 	ljmpl $0x0, $.enter_unreal_mode_unreal
 
@@ -139,6 +143,8 @@ enter_unreal_mode:
 
 	ret
 
+.section .data
+
 hello_msg:
 	.asciz "This is banan-os bootloader"
 

bootloader/bios/command_line.S

@@ -74,6 +74,8 @@ read_user_command_line:
 	ret
 
 
+.section .data
+
 command_line_enter_msg:
 	.asciz "cmdline: "
 

bootloader/bios/disk.S

@@ -470,6 +470,7 @@ print_root_partition_info:
 	popw %ax
 	ret
 
+.section .data
 
 # These will be patched during bootloader installation
 root_disk_guid:

bootloader/bios/elf.S

@@ -196,6 +196,7 @@ elf_read_kernel_to_memory:
 	movw $elf_read_kernel_to_memory_not_loadable_header_msg, %si
 	jmp print_and_halt
 
+.section .data
 
 elf_validate_file_header_invalid_magic_msg:
 	.asciz "ELF: file has invalid ELF magic"

bootloader/bios/ext2.S

@@ -692,6 +692,7 @@ ext2_find_kernel:
 	movw $ext2_kernel_not_reg_msg, %si
 	jmp print_and_halt
 
+.section .data
 
 kernel_path:
 	.short kernel_path1
@@ -704,7 +705,6 @@ kernel_path2:
 	.short 15
 	.asciz "banan-os.kernel"
 
-
 root_partition_does_not_fit_ext2_filesystem_msg:
 	.asciz "Root partition is too small to contain ext2 filesystem"
 root_partition_has_invalid_ext2_magic_msg:

bootloader/bios/framebuffer.S

@@ -185,6 +185,7 @@ vesa_set_video_mode:
 	popw %ax
 	ret
 
+.section .data
 
 vesa_error_msg:
 	.asciz "VESA error"

bootloader/bios/linker.ld

@@ -8,6 +8,8 @@ SECTIONS
 	. = ALIGN(512);
 	stage2_start = .;
 	.stage2 : { *(.stage2) }
+	. = ALIGN(512);
+	.data : { *(.data) }
 	stage2_end = .;
 
 	. = ALIGN(512);

bootloader/bios/memory_map.S

@@ -114,6 +114,7 @@ print_memory_map:
 
 	ret
 
+.section .data
 
 memory_map_msg:
 	.asciz "memmap:"

bootloader/installer/GPT.cpp

@@ -99,33 +99,33 @@ bool GPTFile::install_stage1(std::span<const uint8_t> stage1)
 	return true;
 }
 
-bool GPTFile::install_stage2(std::span<const uint8_t> stage2, const GUID& root_partition_guid)
+bool GPTFile::install_stage2(std::span<const uint8_t> stage2, std::span<const uint8_t> data, const GUID& root_partition_guid)
 {
-	if (stage2.size() < 16)
+	if (data.size() < 16)
 	{
-		std::cerr << m_path << ": contains invalid .stage2 section, too small for patches" << std::endl;
+		std::cerr << m_path << ": contains invalid .data section, too small for patches" << std::endl;
 		return false;
 	}
 
 	// find GUID patch offsets
 	std::size_t disk_guid_offset(-1);
 	std::size_t part_guid_offset(-1);
-	for (std::size_t i = 0; i < stage2.size() - 16; i++)
+	for (std::size_t i = 0; i < data.size() - 16; i++)
 	{
-		if (memcmp(stage2.data() + i, "root disk guid  ", 16) == 0)
+		if (memcmp(data.data() + i, "root disk guid  ", 16) == 0)
 		{
 			if (disk_guid_offset != std::size_t(-1))
 			{
-				std::cerr << m_path << ": contains invalid .stage2 section, multiple patchable disk guids" << std::endl;
+				std::cerr << m_path << ": contains invalid .data section, multiple patchable disk guids" << std::endl;
 				return false;
 			}
 			disk_guid_offset = i;
 		}
-		if (memcmp(stage2.data() + i, "root part guid  ", 16) == 0)
+		if (memcmp(data.data() + i, "root part guid  ", 16) == 0)
 		{
 			if (part_guid_offset != std::size_t(-1))
 			{
-				std::cerr << m_path << ": contains invalid .stage2 section, multiple patchable partition guids" << std::endl;
+				std::cerr << m_path << ": contains invalid .data section, multiple patchable partition guids" << std::endl;
 				return false;
 			}
 			part_guid_offset = i;
@@ -133,15 +133,14 @@ bool GPTFile::install_stage2(std::span<const uint8_t> stage2, const GUID& root_p
 	}
 	if (disk_guid_offset == std::size_t(-1))
 	{
-		std::cerr << m_path << ": contains invalid .stage2 section, no patchable disk guid" << std::endl;
+		std::cerr << m_path << ": contains invalid .data section, no patchable disk guid" << std::endl;
 		return false;
 	}
 	if (part_guid_offset == std::size_t(-1))
 	{
-		std::cerr << m_path << ": contains invalid .stage2 section, no patchable partition guid" << std::endl;
+		std::cerr << m_path << ": contains invalid .data section, no patchable partition guid" << std::endl;
 		return false;
 	}
-	
 
 	auto partition = find_partition_with_type(bios_boot_guid);
 	if (!partition.has_value())
@@ -152,23 +151,28 @@ bool GPTFile::install_stage2(std::span<const uint8_t> stage2, const GUID& root_p
 
 	const std::size_t partition_size = (partition->ending_lba - partition->starting_lba + 1) * SECTOR_SIZE;
 
-	if (stage2.size() > partition_size)
+	std::size_t data_offset = stage2.size();
+	if (std::size_t rem = data_offset % 512)
+		data_offset += 512 - rem;
+
+	if (data_offset + data.size() > partition_size)
 	{
-		std::cerr << m_path << ": can't fit " << stage2.size() << " bytes of data to partition of size " << partition_size << std::endl;
+		std::cerr << m_path << ": can't fit " << stage2.size() + data.size() << " bytes of data to partition of size " << partition_size << std::endl;
 		return false;
 	}
 
 	uint8_t* partition_start = m_mmap + partition->starting_lba * SECTOR_SIZE;
 	memcpy(partition_start, stage2.data(), stage2.size());
+	memcpy(partition_start + data_offset, data.data(), data.size());
 
 	// patch GUIDs
-	*reinterpret_cast<GUID*>(partition_start + disk_guid_offset) = gpt_header().disk_guid;
-	*reinterpret_cast<GUID*>(partition_start + part_guid_offset) = root_partition_guid;
+	*reinterpret_cast<GUID*>(partition_start + data_offset + disk_guid_offset) = gpt_header().disk_guid;
+	*reinterpret_cast<GUID*>(partition_start + data_offset + part_guid_offset) = root_partition_guid;
 
 	return true;
 }
 
-bool GPTFile::install_bootloader(std::span<const uint8_t> stage1, std::span<const uint8_t> stage2, const GUID& root_partition_guid)
+bool GPTFile::install_bootloader(std::span<const uint8_t> stage1, std::span<const uint8_t> stage2, std::span<const uint8_t> data, const GUID& root_partition_guid)
 {
 	if (!find_partition_with_guid(root_partition_guid).has_value())
 	{
@@ -177,7 +181,7 @@ bool GPTFile::install_bootloader(std::span<const uint8_t> stage1, std::span<cons
 	}
 	if (!install_stage1(stage1))
 		return false;
-	if (!install_stage2(stage2, root_partition_guid))
+	if (!install_stage2(stage2, data, root_partition_guid))
 		return false;
 	return true;
 }

bootloader/installer/GPT.h

@@ -65,7 +65,7 @@ public:
 	GPTFile(std::string_view path);
 	~GPTFile();
 
-	bool install_bootloader(std::span<const uint8_t> stage1, std::span<const uint8_t> stage2, const GUID& root_partition_guid);
+	bool install_bootloader(std::span<const uint8_t> stage1, std::span<const uint8_t> stage2, std::span<const uint8_t> data, const GUID& root_partition_guid);
 
 	const GPTHeader& gpt_header() const;
 
@@ -80,7 +80,7 @@ private:
 	std::optional<GPTPartitionEntry> find_partition_with_type(const GUID& type_guid) const;
 
 	bool install_stage1(std::span<const uint8_t> stage1);
-	bool install_stage2(std::span<const uint8_t> stage2, const GUID& root_partition_guid);
+	bool install_stage2(std::span<const uint8_t> stage2, std::span<const uint8_t> data, const GUID& root_partition_guid);
 
 private:
 	const std::string	m_path;

bootloader/installer/build.sh

@@ -1,3 +0,0 @@
-#!/bin/sh
-
-g++ -O2 -std=c++20 main.cpp crc32.cpp ELF.cpp GPT.cpp GUID.cpp -o install-bootloader

bootloader/installer/main.cpp

@@ -26,9 +26,10 @@ int main(int argc, char** argv)
 
 	auto stage1 = bootloader.find_section(".stage1"sv);
 	auto stage2 = bootloader.find_section(".stage2"sv);
-	if (!stage1.has_value() || !stage2.has_value())
+	auto data = bootloader.find_section(".data"sv);
+	if (!stage1.has_value() || !stage2.has_value() || !data.has_value())
 	{
-		std::cerr << bootloader.path() << " doesn't contain .stage1 and .stage2 sections" << std::endl;
+		std::cerr << bootloader.path() << " doesn't contain .stage1, .stage2 and .data sections" << std::endl;
 		return 1;
 	}
 
@@ -36,7 +37,7 @@ int main(int argc, char** argv)
 	if (!disk_image.success())
 		return 1;
 
-	if (!disk_image.install_bootloader(*stage1, *stage2, *root_partition_guid))
+	if (!disk_image.install_bootloader(*stage1, *stage2, *data, *root_partition_guid))
 		return 1;
 	std::cout << "bootloader installed" << std::endl;
 

kernel/CMakeLists.txt

@@ -65,6 +65,9 @@ set(KERNEL_SOURCES
 	kernel/Storage/ATA/ATAController.cpp
 	kernel/Storage/ATA/ATADevice.cpp
 	kernel/Storage/DiskCache.cpp
+	kernel/Storage/NVMe/Controller.cpp
+	kernel/Storage/NVMe/Namespace.cpp
+	kernel/Storage/NVMe/Queue.cpp
 	kernel/Storage/Partition.cpp
 	kernel/Storage/StorageDevice.cpp
 	kernel/Syscall.cpp

kernel/include/kernel/APIC.h

@@ -14,6 +14,9 @@ namespace Kernel
 		virtual void enable_irq(uint8_t) override;
 		virtual bool is_in_service(uint8_t) override;
 
+		virtual BAN::ErrorOr<void> reserve_irq(uint8_t irq) override;
+		virtual BAN::Optional<uint8_t> get_free_irq() override;
+
 	private:
 		uint32_t read_from_local_apic(ptrdiff_t);
 		void write_to_local_apic(ptrdiff_t, uint32_t);
@@ -54,6 +57,7 @@ namespace Kernel
 		Kernel::vaddr_t			m_local_apic_vaddr = 0;
 		BAN::Vector<IOAPIC>		m_io_apics;	
 		uint8_t					m_irq_overrides[0x100] {};
+		uint8_t					m_reserved_gsis[0x100 / 8] {};
 	};
 
 }

kernel/include/kernel/Input/KeyEvent.h

@@ -33,19 +33,6 @@ namespace Kernel::Input
 		return ((row + 1) << 5) | (col + 0b11111 - 8);
 	}
 
-	enum ModifierKeycode
-	{
-		CapsLock	= keycode_normal(2, 0),
-		NumLock		= keycode_numpad(0, 0),
-		ScrollLock	= keycode_function(20),
-		LShift		= keycode_normal(3, 0),
-		RShift		= keycode_normal(3, 12),
-		LCtrl		= keycode_normal(4, 0),
-		RCtrl		= keycode_normal(4, 5),
-		LAlt		= keycode_normal(4, 2),
-		RAlt		= keycode_normal(4, 4),
-	};
-
 	enum class Key
 	{
 		Invalid, None,
@@ -59,7 +46,7 @@ namespace Kernel::Input
 		Equals, QuestionMark, Plus, BackSlash, Acute, BackTick, TwoDots, Cedilla, Backspace, AtSign, Pound, Dollar, Euro,
 		Escape, Tab, CapsLock, LeftShift, LeftCtrl, Super, LeftAlt, RightAlt, AltGr = RightAlt, RightCtrl, RightShift,
 		SingleQuote, Asterix, Caret, Tilde, ArrowUp, ArrowDown, ArrowLeft, ArrowRight,
-		Comma, Semicolon, Period, Colon, Hyphen, Underscore, NumLock, ScrollLock, LessThan, GreaterThan, Pipe,
+		Comma, Semicolon, Period, Colon, Hyphen, Underscore, NumLock, ScrollLock, LessThan, GreaterThan, Pipe, Negation, BrokenBar,
 		Numpad0, Numpad1, Numpad2, Numpad3, Numpad4, Numpad5, Numpad6, Numpad7, Numpad8, Numpad9,
 		NumpadPlus, NumpadMinus, NumpadMultiply, NumpadDivide, NumpadEnter, NumpadDecimal,
 		VolumeMute, VolumeUp, VolumeDown, Calculator, MediaPlayPause, MediaStop, MediaPrevious, MediaNext,

kernel/include/kernel/Input/PS2/Controller.h

@@ -38,6 +38,8 @@ namespace Kernel::Input
 		BAN::ErrorOr<void> device_send_byte(uint8_t device_index, uint8_t byte);
 		BAN::ErrorOr<void> device_send_byte_and_wait_ack(uint8_t device_index, uint8_t byte);
 
+		uint8_t get_device_index(PS2Device*) const;
+
 	private:
 		struct Command
 		{

kernel/include/kernel/InterruptController.h

@@ -1,5 +1,8 @@
 #pragma once
 
+#include <BAN/Optional.h>
+#include <BAN/Errors.h>
+
 #include <stdint.h>
 
 #define DISABLE_INTERRUPTS() asm volatile("cli")
@@ -37,6 +40,9 @@ namespace Kernel
 		static void initialize(bool force_pic);
 		static InterruptController& get();
 
+		virtual BAN::ErrorOr<void> reserve_irq(uint8_t irq) = 0;
+		virtual BAN::Optional<uint8_t> get_free_irq() = 0;
+
 		bool is_using_apic() const { return m_using_apic; }
 
 		void enter_acpi_mode();

kernel/include/kernel/Memory/Types.h

@@ -11,6 +11,7 @@
 #endif
 
 #define PAGE_SIZE ((uintptr_t)4096)
+#define PAGE_SIZE_SHIFT 12
 #define PAGE_ADDR_MASK (~(uintptr_t)0xFFF)
 
 namespace Kernel

kernel/include/kernel/PCI.h

@@ -79,7 +79,8 @@ namespace Kernel::PCI
 		uint16_t vendor_id() const { return m_vendor_id; }
 		uint16_t device_id() const { return m_device_id; }
 
-		BAN::ErrorOr<uint8_t> get_irq();
+		BAN::ErrorOr<void> reserve_irqs(uint8_t count);
+		uint8_t get_irq(uint8_t index);
 
 		BAN::ErrorOr<BAN::UniqPtr<BarRegion>> allocate_bar_region(uint8_t bar_num);
 
@@ -92,15 +93,15 @@ namespace Kernel::PCI
 		void enable_io_space();
 		void disable_io_space();
 
-		void enable_pin_interrupts();
-		void disable_pin_interrupts();
-
 	private:
 		void enumerate_capabilites();
 
 		void set_command_bits(uint16_t mask);
 		void unset_command_bits(uint16_t mask);
 
+		void enable_pin_interrupts();
+		void disable_pin_interrupts();
+
 	private:
 		const uint8_t m_bus;
 		const uint8_t m_dev;
@@ -114,6 +115,9 @@ namespace Kernel::PCI
 		uint16_t m_vendor_id;
 		uint16_t m_device_id;
 
+		uint32_t m_reserved_irqs { 0 };
+		uint8_t m_reserved_irq_count { 0 };
+
 		BAN::Optional<uint8_t> m_offset_msi;
 		BAN::Optional<uint8_t> m_offset_msi_x;
 	};

kernel/include/kernel/PIC.h

@@ -12,12 +12,16 @@ namespace Kernel
 		virtual void enable_irq(uint8_t) override;
 		virtual bool is_in_service(uint8_t) override;
 
+		virtual BAN::ErrorOr<void> reserve_irq(uint8_t irq) override;
+		virtual BAN::Optional<uint8_t> get_free_irq() override;
+
 		static void remap();
 		static void mask_all();
 
 	private:
 		static PIC* create();
 		friend class InterruptController;
+		uint16_t m_reserved_irqs { 0 };
 	};
 
 }

kernel/include/kernel/Storage/NVMe/Controller.h

@@ -0,0 +1,52 @@
+#pragma once
+
+#include <BAN/Vector.h>
+#include <kernel/InterruptController.h>
+#include <kernel/PCI.h>
+#include <kernel/Storage/NVMe/Definitions.h>
+#include <kernel/Storage/NVMe/Namespace.h>
+#include <kernel/Storage/NVMe/Queue.h>
+
+namespace Kernel
+{
+
+	class NVMeController final : public StorageController, public CharacterDevice
+	{
+		BAN_NON_COPYABLE(NVMeController);
+		BAN_NON_MOVABLE(NVMeController);
+
+	public:
+		static BAN::ErrorOr<BAN::RefPtr<StorageController>> create(PCI::Device&);
+		~NVMeController() { ASSERT_NOT_REACHED(); }
+
+		NVMeQueue& io_queue() { return *m_io_queue; }
+
+		virtual dev_t rdev() const override { return m_rdev; }
+		virtual BAN::StringView name() const override { return m_name; }
+
+	private:
+		NVMeController(PCI::Device& pci_device);
+		virtual BAN::ErrorOr<void> initialize() override;
+
+		BAN::ErrorOr<void> identify_controller();
+		BAN::ErrorOr<void> identify_namespaces();
+
+		BAN::ErrorOr<void> wait_until_ready(bool expected_value);
+		BAN::ErrorOr<void> create_admin_queue();
+		BAN::ErrorOr<void> create_io_queue();
+
+	private:
+		PCI::Device& m_pci_device;
+		BAN::UniqPtr<PCI::BarRegion> m_bar0;
+		volatile NVMe::ControllerRegisters* m_controller_registers;
+
+		BAN::UniqPtr<NVMeQueue> m_admin_queue;
+		BAN::UniqPtr<NVMeQueue> m_io_queue;
+
+		BAN::Vector<BAN::RefPtr<NVMeNamespace>> m_namespaces;
+
+		char m_name[20];
+		const dev_t m_rdev;
+	};
+
+}

kernel/include/kernel/Storage/NVMe/Definitions.h

@@ -0,0 +1,295 @@
+#pragma once
+
+#include <stdint.h>
+
+namespace Kernel::NVMe
+{
+
+	struct CAP
+	{
+		uint64_t mqes : 16;
+		uint64_t cqr : 1;
+		uint64_t ams : 2;
+		uint64_t __reserved0 : 5;
+		uint64_t to : 8;
+		uint64_t dstrd : 4;
+		uint64_t nssrs : 1;
+		uint64_t css : 8;
+		uint64_t bps : 1;
+		uint64_t cps : 2;
+		uint64_t mpsmin : 4;
+		uint64_t mpsmax : 4;
+		uint64_t pmrs : 1;
+		uint64_t cmpbs : 1;
+		uint64_t nsss : 1;
+		uint64_t crms : 2;
+		uint64_t __reserved1 : 3;
+	};
+	static_assert(sizeof(CAP) == sizeof(uint64_t));
+
+	enum CAP_CSS
+	{
+		CAP_CSS_NVME  = 1 << 0,
+		CAP_CSS_IO    = 1 << 6,
+		CAP_CSS_ADMIN = 1 << 7,
+	};
+
+	struct VS
+	{
+		uint32_t tertiary : 8;
+		uint32_t minor : 8;
+		uint32_t major : 16;
+	};
+	static_assert(sizeof(VS) == sizeof(uint32_t));
+
+	struct CC
+	{
+		uint32_t en : 1;
+		uint32_t __reserved0 : 3;
+		uint32_t css : 3;
+		uint32_t mps : 4;
+		uint32_t ams : 3;
+		uint32_t shn : 2;
+		uint32_t iosqes : 4;
+		uint32_t iocqes : 4;
+		uint32_t crime : 1;
+		uint32_t __reserved1 : 7;
+	};
+	static_assert(sizeof(CC) == sizeof(uint32_t));
+
+	struct CSTS
+	{
+		uint32_t rdy : 1;
+		uint32_t cfs : 1;
+		uint32_t shts : 2;
+		uint32_t nssro : 1;
+		uint32_t pp : 1;
+		uint32_t st : 1;
+		uint32_t __reserved : 25;
+	};
+	static_assert(sizeof(CSTS) == sizeof(uint32_t));
+
+	struct AQA
+	{
+		uint32_t asqs : 12;
+		uint32_t __reserved0 : 4;
+		uint32_t acqs : 12;
+		uint32_t __reserved1 : 4;
+	};
+	static_assert(sizeof(AQA) == sizeof(uint32_t));
+
+	// BAR0
+	struct ControllerRegisters
+	{
+		CAP cap;
+		VS vs;
+		uint32_t intms;
+		uint32_t intmc;
+		CC cc;
+		uint8_t __reserved0[4];
+		CSTS csts;
+		uint32_t nssr;
+		AQA aqa;
+		uint64_t asq;
+		uint64_t acq;
+
+		static constexpr uint32_t SQ0TDBL = 0x1000;
+	};
+	static_assert(sizeof(ControllerRegisters) == 0x38);
+
+	struct DoorbellRegisters
+	{
+		uint32_t sq_tail;
+		uint32_t cq_head;
+	} __attribute__((packed));
+
+	struct CompletionQueueEntry
+	{
+		uint32_t dontcare[3];
+		uint16_t cid;
+		uint16_t sts;
+	} __attribute__((packed));
+	static_assert(sizeof(CompletionQueueEntry) == 16);
+
+	struct DataPtr
+	{
+		union
+		{
+			struct
+			{
+				uint64_t prp1;
+				uint64_t prp2;
+			};
+			uint8_t sgl1[16];
+		};
+	};
+
+	struct CommandGeneric
+	{
+		uint32_t nsid;
+		uint32_t cdw2;
+		uint32_t cdw3;
+		uint64_t mptr;
+		DataPtr dptr;
+		uint32_t cdw10;
+		uint32_t cdw11;
+		uint32_t cdw12;
+		uint32_t cdw13;
+		uint32_t cdw14;
+		uint32_t cdw15;
+	} __attribute__((packed));
+	static_assert(sizeof(CommandGeneric) == 15 * sizeof(uint32_t));
+
+	struct CommandIdentify
+	{
+		uint32_t nsid;
+		uint64_t __reserved0[2];
+		DataPtr dptr;
+		// dword 10
+		uint8_t cns;
+		uint8_t __reserved1;
+		uint16_t cntid;
+		// dword 11 
+		uint16_t cnsid;
+		uint8_t __reserved2;
+		uint8_t csi;
+		// dword 12-15
+		uint32_t __reserved3[4];
+	} __attribute__((packed));
+	static_assert(sizeof(CommandIdentify) == 15 * sizeof(uint32_t));
+
+	struct CommandCreateCQ
+	{
+		uint32_t __reserved0;
+		uint64_t __reserved1[2];
+		DataPtr dptr;
+		// dword 10
+		uint16_t qid;
+		uint16_t qsize;
+		// dword 11
+		uint16_t pc : 1;
+		uint16_t ien : 1;
+		uint16_t __reserved2 : 14;
+		uint16_t iv;
+		// dword 12-15
+		uint32_t __reserved4[4];
+	} __attribute__((packed));
+	static_assert(sizeof(CommandCreateCQ) == 15 * sizeof(uint32_t));
+
+	struct CommandCreateSQ
+	{
+		uint32_t __reserved0;
+		uint64_t __reserved1[2];
+		DataPtr dptr;
+		// dword 10
+		uint16_t qid;
+		uint16_t qsize;
+		// dword 11
+		uint16_t pc : 1;
+		uint16_t qprio : 2;
+		uint16_t __reserved2 : 13;
+		uint16_t cqid;
+		// dword 12
+		uint16_t nvmsetid;
+		uint16_t __reserved4;
+		// dword 13-15
+		uint32_t __reserved5[3];
+	} __attribute__((packed));
+	static_assert(sizeof(CommandCreateSQ) == 15 * sizeof(uint32_t));
+
+
+	struct CommandRead
+	{
+		uint32_t nsid;
+		uint64_t __reserved0;
+		uint64_t mptr;
+		DataPtr dptr;
+		// dword 10-11
+		uint64_t slba;
+		// dword 12
+		uint16_t nlb;
+		uint16_t __reserved1;
+		// dword 13-15
+		uint32_t __reserved2[3];
+	} __attribute__((packed));
+	static_assert(sizeof(CommandRead) == 15 * sizeof(uint32_t));
+
+	struct SubmissionQueueEntry
+	{
+		uint8_t opc;
+		uint8_t fuse : 2;
+		uint8_t __reserved : 4;
+		uint8_t psdt : 2;
+		uint16_t cid;
+		union
+		{
+			CommandGeneric generic;
+			CommandIdentify identify;
+			CommandCreateCQ create_cq;
+			CommandCreateSQ create_sq;
+			CommandRead read;
+		};
+	} __attribute__((packed));
+	static_assert(sizeof(SubmissionQueueEntry) == 64);
+
+	enum OPC : uint8_t
+	{
+		OPC_ADMIN_CREATE_SQ = 0x01,
+		OPC_ADMIN_CREATE_CQ = 0x05,
+		OPC_ADMIN_IDENTIFY = 0x06,
+		OPC_IO_WRITE = 0x01,
+		OPC_IO_READ = 0x02,
+	};
+
+	enum CNS : uint8_t
+	{
+		CNS_INDENTIFY_NAMESPACE = 0x00,
+		CNS_INDENTIFY_CONTROLLER = 0x01,
+		CNS_INDENTIFY_ACTIVE_NAMESPACES = 0x02,
+	};
+
+	struct NamespaceIdentify
+	{
+		uint64_t nsze;
+		uint64_t ncap;
+		uint64_t nuse;
+		uint8_t nsfeat;
+		uint8_t nlbaf;
+		uint8_t flbas;
+		uint8_t mc;
+		uint8_t dpc;
+		uint8_t dps;
+		uint8_t nmic;
+		uint8_t rescap;
+		uint8_t fpi;
+		uint8_t dlfeat;
+		uint16_t nawun;
+		uint16_t nawupf;
+		uint16_t nacwu;
+		uint16_t nabsn;
+		uint16_t nabo;
+		uint16_t nabspf;
+		uint16_t noiob;
+		uint64_t nvmcap[2];
+		uint16_t npwg;
+		uint16_t npwa;
+		uint16_t npdg;
+		uint16_t npda;
+		uint16_t nows;
+		uint16_t mssrl;
+		uint32_t mcl;
+		uint8_t msrc;
+		uint8_t __reserved0[11];
+		uint32_t adagrpid;
+		uint8_t __reserved1[3];
+		uint8_t nsattr;
+		uint16_t nvmsetid;
+		uint16_t endgid;
+		uint64_t nguid[2];
+		uint64_t eui64;
+		uint32_t lbafN[64];
+		uint8_t vendor_specific[3712];
+	} __attribute__((packed));
+	static_assert(sizeof(NamespaceIdentify) == 0x1000);
+
+}

kernel/include/kernel/Storage/NVMe/Namespace.h

@@ -0,0 +1,41 @@
+#pragma once
+
+#include <kernel/Memory/DMARegion.h>
+#include <kernel/Storage/StorageDevice.h>
+
+namespace Kernel
+{
+
+	class NVMeController;
+
+	class NVMeNamespace : public StorageDevice
+	{
+	public:
+		static BAN::ErrorOr<BAN::RefPtr<NVMeNamespace>> create(NVMeController&, uint32_t nsid, uint64_t block_count, uint32_t block_size);
+
+		virtual uint32_t sector_size() const override { return m_block_size; }
+		virtual uint64_t total_size() const override { return m_block_size * m_block_count; }
+
+		virtual dev_t rdev() const override { return m_rdev; }
+		virtual BAN::StringView name() const { return m_name; }
+
+	private:
+		NVMeNamespace(NVMeController&, uint32_t nsid, uint64_t block_count, uint32_t block_size);
+		BAN::ErrorOr<void> initialize();
+
+		virtual BAN::ErrorOr<void> read_sectors_impl(uint64_t lba, uint64_t sector_count, BAN::ByteSpan) override;
+		virtual BAN::ErrorOr<void> write_sectors_impl(uint64_t lba, uint64_t sector_count, BAN::ConstByteSpan) override;
+
+	private:
+		NVMeController& m_controller;
+		BAN::UniqPtr<DMARegion> m_dma_region;
+
+		const uint32_t m_nsid;
+		const uint32_t m_block_size;
+		const uint64_t m_block_count;
+
+		char m_name[10] {};
+		const dev_t m_rdev;
+	};
+
+}

kernel/include/kernel/Storage/NVMe/Queue.h

@@ -0,0 +1,37 @@
+#pragma once
+
+#include <BAN/UniqPtr.h>
+#include <BAN/Vector.h>
+#include <kernel/InterruptController.h>
+#include <kernel/Memory/DMARegion.h>
+#include <kernel/Semaphore.h>
+#include <kernel/Storage/NVMe/Definitions.h>
+
+namespace Kernel
+{
+
+	class NVMeQueue : public Interruptable
+	{
+	public:
+		NVMeQueue(BAN::UniqPtr<Kernel::DMARegion>&& cq, BAN::UniqPtr<Kernel::DMARegion>&& sq, volatile NVMe::DoorbellRegisters& db, uint32_t qdepth, uint8_t irq);
+
+		uint16_t submit_command(NVMe::SubmissionQueueEntry& sqe);
+
+		virtual void handle_irq() final override;
+
+	private:
+		SpinLock m_lock;
+		BAN::UniqPtr<Kernel::DMARegion> m_completion_queue;
+		BAN::UniqPtr<Kernel::DMARegion> m_submission_queue;
+		volatile NVMe::DoorbellRegisters& m_doorbell;
+		const uint32_t m_qdepth;
+		uint32_t m_sq_tail { 0 };
+		uint32_t m_cq_head { 0 };
+		uint16_t m_cq_valid_phase { 1 };
+
+		Semaphore			m_semaphore;
+		volatile uint16_t	m_status;
+		volatile bool		m_done { false };
+	};
+
+}

kernel/include/kernel/Storage/Partition.h

@@ -9,7 +9,7 @@ namespace Kernel
 	class Partition final : public BlockDevice
 	{
 	public:
-		static BAN::ErrorOr<BAN::RefPtr<Partition>> create(BAN::RefPtr<BlockDevice>, const BAN::GUID& type, const BAN::GUID& guid, uint64_t first_block, uint64_t last_block, uint64_t attr, const char* label, uint32_t index);
+		static BAN::ErrorOr<BAN::RefPtr<Partition>> create(BAN::RefPtr<BlockDevice>, const BAN::GUID& type, const BAN::GUID& guid, uint64_t first_block, uint64_t last_block, uint64_t attr, const char* label, uint32_t index, BAN::StringView name_prefix);
 
 		const BAN::GUID& partition_type() const { return m_type; }
 		const BAN::GUID& partition_guid() const { return m_guid; }
@@ -26,7 +26,7 @@ namespace Kernel
 		virtual BAN::StringView name() const override { return m_name; }
 
 	private:
-		Partition(BAN::RefPtr<BlockDevice>, const BAN::GUID&, const BAN::GUID&, uint64_t, uint64_t, uint64_t, const char*, uint32_t);
+		Partition(BAN::RefPtr<BlockDevice>, const BAN::GUID&, const BAN::GUID&, uint64_t, uint64_t, uint64_t, const char*, uint32_t, BAN::StringView);
 
 	private:
 		BAN::RefPtr<BlockDevice> m_device;

kernel/include/kernel/Storage/StorageDevice.h

@@ -16,7 +16,7 @@ namespace Kernel
 		{ }
 		virtual ~StorageDevice();
 
-		BAN::ErrorOr<void> initialize_partitions();
+		BAN::ErrorOr<void> initialize_partitions(BAN::StringView name_prefix);
 
 		virtual BAN::ErrorOr<void> read_blocks(uint64_t lba, size_t sector_count, BAN::ByteSpan buffer) override		{ return read_sectors(lba, sector_count, buffer); }
 		virtual BAN::ErrorOr<void> write_blocks(uint64_t lba, size_t sector_count, BAN::ConstByteSpan buffer) override	{ return write_sectors(lba, sector_count, buffer); }

kernel/kernel/APIC.cpp

@@ -223,8 +223,16 @@ namespace Kernel
 
 	void APIC::enable_irq(uint8_t irq)
 	{
+		CriticalScope _;
+
 		uint32_t gsi = m_irq_overrides[irq];
 
+		{
+			int byte = gsi / 8;
+			int bit  = gsi % 8;
+			ASSERT(m_reserved_gsis[byte] & (1 << bit));
+		}
+
 		IOAPIC* ioapic = nullptr;
 		for (IOAPIC& io : m_io_apics)
 		{
@@ -258,4 +266,67 @@ namespace Kernel
 		return isr & (1 << bit);
 	}
 
+	BAN::ErrorOr<void> APIC::reserve_irq(uint8_t irq)
+	{
+		CriticalScope _;
+
+		uint32_t gsi = m_irq_overrides[irq];
+
+		IOAPIC* ioapic = nullptr;
+		for (IOAPIC& io : m_io_apics)
+		{
+			if (io.gsi_base <= gsi && gsi <= io.gsi_base + io.max_redirs)
+			{
+				ioapic = &io;
+				break;
+			}
+		}
+
+		if (!ioapic)
+		{
+			dwarnln("Cannot enable irq {} for APIC", irq);
+			return BAN::Error::from_errno(EFAULT);
+		}
+
+		int byte = gsi / 8;
+		int bit  = gsi % 8;
+		if (m_reserved_gsis[byte] & (1 << bit))
+		{
+			dwarnln("irq {} is already reserved", irq);
+			return BAN::Error::from_errno(EFAULT);
+		}
+		m_reserved_gsis[byte] |= 1 << bit;
+		return {};
+	}
+
+	BAN::Optional<uint8_t> APIC::get_free_irq()
+	{
+		CriticalScope _;
+		for (int irq = 0; irq <= 0xFF; irq++)
+		{
+			uint32_t gsi = m_irq_overrides[irq];
+		
+			IOAPIC* ioapic = nullptr;
+			for (IOAPIC& io : m_io_apics)
+			{
+				if (io.gsi_base <= gsi && gsi <= io.gsi_base + io.max_redirs)
+				{
+					ioapic = &io;
+					break;
+				}
+			}
+
+			if (!ioapic)
+				continue;
+
+			int byte = gsi / 8;
+			int bit  = gsi % 8;
+			if (m_reserved_gsis[byte] & (1 << bit))
+				continue;
+			m_reserved_gsis[byte] |= 1 << bit;
+			return irq;
+		}
+		return {};
+	}
+
 }

kernel/kernel/Input/KeyEvent.cpp

@@ -24,7 +24,7 @@ namespace Kernel::Input
 			"=", "?", "+", "\\", "´", "`", "¨", "¸", nullptr, "@", "£", "$", "€",
 			nullptr, "\t", nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr,
 			"'", "*", "^", "~", nullptr, nullptr, nullptr, nullptr,
-			",", ";", ".", ":", "-", "_", nullptr, nullptr, "<", ">", "|",
+			",", ";", ".", ":", "-", "_", nullptr, nullptr, "<", ">", "|", "¬", "¦",
 			"0", "1", "2", "3", "4", "5", "6", "7", "8", "9",
 			"+", "-", "*", "/", nullptr, ",",
 			nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr,
@@ -43,7 +43,7 @@ namespace Kernel::Input
 			"=", "?", "+", "\\", "´", "`", "¨", "¸", nullptr, "@", "£", "$", "€",
 			nullptr, "\t", nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr,
 			"'", "*", "^", "~", nullptr, nullptr, nullptr, nullptr,
-			",", ";", ".", ":", "-", "_", nullptr, nullptr, "<", ">", "|",
+			",", ";", ".", ":", "-", "_", nullptr, nullptr, "<", ">", "|", "¬", "¦",
 			"0", "1", "2", "3", "4", "5", "6", "7", "8", "9",
 			"+", "-", "*", "/", nullptr, ",",
 			nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr,

kernel/kernel/Input/KeyboardLayout.cpp

@@ -1,4 +1,5 @@
 #include <BAN/HashMap.h>
+#include <kernel/CriticalScope.h>
 #include <kernel/FS/VirtualFileSystem.h>
 #include <kernel/Input/KeyboardLayout.h>
 
@@ -106,18 +107,15 @@ namespace Kernel::Input
 		return {};
 	}
 
-	BAN::ErrorOr<void> KeyboardLayout::load_from_file(BAN::StringView path)
+	static BAN::ErrorOr<BAN::Vector<BAN::String>> load_keymap_lines_and_parse_includes(BAN::StringView path)
 	{
-		if (s_name_to_key.empty())
-			TRY(initialize_name_to_key());
-
-		auto inode = TRY(VirtualFileSystem::get().file_from_absolute_path({ 0, 0, 0, 0 }, path, 0)).inode;
+		auto file = TRY(VirtualFileSystem::get().file_from_absolute_path({ 0, 0, 0, 0 }, path, 0));
 
 		BAN::String file_data;
-		TRY(file_data.resize(inode->size()));
-		TRY(inode->read(0, { reinterpret_cast<uint8_t*>(file_data.data()), file_data.size() }));
+		TRY(file_data.resize(file.inode->size()));
+		TRY(file.inode->read(0, BAN::ByteSpan { reinterpret_cast<uint8_t*>(file_data.data()), file_data.size() }));
 
-		auto new_layout = TRY(BAN::UniqPtr<KeyboardLayout>::create());
+		BAN::Vector<BAN::String> result;
 
 		auto lines = TRY(file_data.sv().split('\n'));
 		for (auto line : lines)
@@ -126,10 +124,89 @@ namespace Kernel::Input
 			if (parts.empty() || parts.front().front() == '#')
 				continue;
 
+			if (parts.front() == "include"sv)
+			{
+				if (parts.size() != 2)
+				{
+					dprintln("Invalid modifier instruction in keymap '{}'", line);
+					dprintln("  format: include \"PATH\"");
+					return BAN::Error::from_errno(EINVAL);
+				}
+
+				if (parts[1].size() < 2 || parts[1].front() != '"' || parts[1].back() != '"')
+				{
+					dprintln("Invalid modifier instruction in keymap '{}'", line);
+					dprintln("  format: include \"PATH\"");
+					return BAN::Error::from_errno(EINVAL);	
+				}
+				parts[1] = parts[1].substring(1, parts[1].size() - 2);
+
+				BAN::String include_path;
+				TRY(include_path.append(file.canonical_path));
+				ASSERT(include_path.sv().contains('/'));
+				while (include_path.back() != '/')
+					include_path.pop_back();
+				TRY(include_path.append(parts[1]));
+
+				auto new_lines = TRY(load_keymap_lines_and_parse_includes(include_path));
+				TRY(result.reserve(result.size() + new_lines.size()));
+				for (auto& line : new_lines)
+					TRY(result.push_back(BAN::move(line)));
+			}
+			else
+			{
+				BAN::String line_str;
+				TRY(line_str.append(line));
+				TRY(result.push_back(BAN::move(line_str)));
+			}
+		}
+
+		return result;
+	}
+
+	BAN::ErrorOr<void> KeyboardLayout::load_from_file(BAN::StringView path)
+	{
+		if (s_name_to_key.empty())
+			TRY(initialize_name_to_key());
+
+		auto new_layout = TRY(BAN::UniqPtr<KeyboardLayout>::create());
+
+		bool shift_is_mod = false;
+		bool altgr_is_mod = false;
+
+		auto lines = TRY(load_keymap_lines_and_parse_includes(path));
+		for (const auto& line : lines)
+		{
+			auto parts = TRY(line.sv().split([](char c) -> bool { return isspace(c); }));
+			if (parts.empty() || parts.front().front() == '#')
+				continue;
+
 			if (parts.size() == 1)
 			{
 				dprintln("Invalid line in keymap '{}'", line);
 				dprintln("  format: KEYCODE KEY [MODIFIER=KEY]...");
+				dprintln("  format: mod MODIFIER");
+				dprintln("  format: include \"PATH\"");
+				return BAN::Error::from_errno(EINVAL);
+			}
+
+			if (parts.front() == "mod"sv)
+			{
+				if (parts.size() != 2)
+				{
+					dprintln("Invalid modifier instruction in keymap '{}'", line);
+					dprintln("  format: mod MODIFIER");
+					return BAN::Error::from_errno(EINVAL);
+				}
+				if (parts[1] == "shift"sv)
+					shift_is_mod = true;
+				else if (parts[1] == "altgr"sv)
+					altgr_is_mod = true;
+				else
+				{
+					dprintln("Unrecognized modifier '{}'", parts[1]);
+					return BAN::Error::from_errno(EINVAL);
+				}
 				continue;
 			}
 
@@ -137,14 +214,14 @@ namespace Kernel::Input
 			if (!keycode.has_value())
 			{
 				dprintln("Invalid keycode '{}', keycode must number between [0, 0xFF[", parts.front());
-				continue;
+				return BAN::Error::from_errno(EINVAL);
 			}
 
 			auto default_key = parse_key(parts[1]);
 			if (!default_key.has_value())
 			{
 				dprintln("Unrecognized key '{}'", parts[1]);
-				continue;
+				return BAN::Error::from_errno(EINVAL);
 			}
 
 			new_layout->m_keycode_to_key_normal[*keycode] = *default_key;
@@ -157,31 +234,41 @@ namespace Kernel::Input
 				if (pair.size() != 2)
 				{
 					dprintln("Invalid modifier format '{}', modifier format: MODIFIRER=KEY", parts[i]);
-					continue;
+					return BAN::Error::from_errno(EINVAL);
 				}
 
 				auto key = parse_key(pair.back());
 				if (!key.has_value())
 				{
 					dprintln("Unrecognized key '{}'", pair.back());
-					continue;
+					return BAN::Error::from_errno(EINVAL);
 				}
 
-				if (pair.front() == "shift"sv)
+				if (shift_is_mod && pair.front() == "shift"sv)
 					new_layout->m_keycode_to_key_shift[*keycode] = *key;
-				else if (pair.front() == "altgr"sv)
+				else if (altgr_is_mod && pair.front() == "altgr"sv)
 					new_layout->m_keycode_to_key_altgr[*keycode] = *key;
 				else
 				{
 					dprintln("Unrecognized modifier '{}'", pair.front());
-					continue;
+					return BAN::Error::from_errno(EINVAL);
 				}
 			}
 		}
 
-		m_keycode_to_key_normal = new_layout->m_keycode_to_key_normal;
-		m_keycode_to_key_shift  = new_layout->m_keycode_to_key_shift;
-		m_keycode_to_key_altgr  = new_layout->m_keycode_to_key_altgr;
+		CriticalScope _;
+
+		for (size_t i = 0; i < new_layout->m_keycode_to_key_normal.size(); i++)
+			if (new_layout->m_keycode_to_key_normal[i] != Key::None)
+				m_keycode_to_key_normal[i] = new_layout->m_keycode_to_key_normal[i];
+
+		for (size_t i = 0; i < new_layout->m_keycode_to_key_shift.size(); i++)
+			if (new_layout->m_keycode_to_key_shift[i] != Key::None)
+				m_keycode_to_key_shift[i] = new_layout->m_keycode_to_key_shift[i];
+
+		for (size_t i = 0; i < new_layout->m_keycode_to_key_altgr.size(); i++)
+			if (new_layout->m_keycode_to_key_altgr[i] != Key::None)
+				m_keycode_to_key_altgr[i] = new_layout->m_keycode_to_key_altgr[i];
 
 		return {};
 	}
@@ -205,6 +292,7 @@ namespace Kernel::Input
 		TRY(s_name_to_key.insert("BackSlash"sv,				Key::BackSlash));
 		TRY(s_name_to_key.insert("Backspace"sv,				Key::Backspace));
 		TRY(s_name_to_key.insert("BackTick"sv,				Key::BackTick));
+		TRY(s_name_to_key.insert("BrokenBar"sv,				Key::BrokenBar));
 		TRY(s_name_to_key.insert("C"sv,						Key::C));
 		TRY(s_name_to_key.insert("Calculator"sv,			Key::Calculator));
 		TRY(s_name_to_key.insert("CapsLock"sv,				Key::CapsLock));
@@ -276,6 +364,7 @@ namespace Kernel::Input
 		TRY(s_name_to_key.insert("MediaPrevious"sv,			Key::MediaPrevious));
 		TRY(s_name_to_key.insert("MediaStop"sv,				Key::MediaStop));
 		TRY(s_name_to_key.insert("N"sv,						Key::N));
+		TRY(s_name_to_key.insert("Negation"sv,				Key::Negation));
 		TRY(s_name_to_key.insert("None"sv,					Key::None));
 		TRY(s_name_to_key.insert("NumLock"sv,				Key::NumLock));
 		TRY(s_name_to_key.insert("Numpad0"sv,				Key::Numpad0));

kernel/kernel/Input/PS2/Controller.cpp

@@ -89,11 +89,20 @@ namespace Kernel::Input
 		return {};
 	}
 
+	uint8_t PS2Controller::get_device_index(PS2Device* device) const
+	{
+		ASSERT(device);
+		if (m_devices[0] && device == m_devices[0].ptr())
+			return 0;
+		if (m_devices[1] && device == m_devices[1].ptr())
+			return 1;
+		ASSERT_NOT_REACHED();
+	}
+
 	bool PS2Controller::append_command_queue(PS2Device* device, uint8_t command, uint8_t response_size)
 	{
 		// NOTE: command queue push/pop must be done without interrupts
 		CriticalScope _;
-		ASSERT(device && (device == m_devices[0].ptr() || device == m_devices[1].ptr()));
 		if (m_command_queue.size() + 1 >= m_command_queue.capacity())
 		{
 			dprintln("PS/2 command queue full");
@@ -101,7 +110,7 @@ namespace Kernel::Input
 		}
 		m_command_queue.push(Command {
 			.state			= Command::State::NotSent,
-			.device_index	= (device == m_devices[0].ptr()) ? uint8_t(0) : uint8_t(1),
+			.device_index	= get_device_index(device),
 			.out_data		= { command, 0x00 },
 			.out_count		= 1,
 			.in_count		= response_size,
@@ -114,7 +123,6 @@ namespace Kernel::Input
 	{
 		// NOTE: command queue push/pop must be done without interrupts
 		CriticalScope _;
-		ASSERT(device && (device == m_devices[0].ptr() || device == m_devices[1].ptr()));
 		if (m_command_queue.size() + 1 >= m_command_queue.capacity())
 		{
 			dprintln("PS/2 command queue full");
@@ -122,7 +130,7 @@ namespace Kernel::Input
 		}
 		m_command_queue.push(Command {
 			.state			= Command::State::NotSent,
-			.device_index	= (device == m_devices[0].ptr()) ? uint8_t(0) : uint8_t(1),
+			.device_index	= get_device_index(device),
 			.out_data		= { command, data },
 			.out_count		= 2,
 			.in_count		= response_size,
@@ -167,8 +175,7 @@ namespace Kernel::Input
 			return false;
 		auto& command = m_command_queue.front();
 
-		ASSERT(device && (device == m_devices[0].ptr() || device == m_devices[1].ptr()));
-		if (command.device_index != (device == m_devices[0].ptr()) ? 0 : 1)
+		if (command.device_index != get_device_index(device))
 			return false;
 
 		switch (command.state)
@@ -322,6 +329,18 @@ namespace Kernel::Input
 			}
 		}
 
+		// Reserve IRQs
+		if (m_devices[0] && InterruptController::get().reserve_irq(PS2::IRQ::DEVICE0).is_error())
+		{
+			dwarnln("Could not reserve irq for PS/2 port 1");
+			m_devices[0].clear();
+		}
+		if (m_devices[1] && InterruptController::get().reserve_irq(PS2::IRQ::DEVICE1).is_error())
+		{
+			dwarnln("Could not reserve irq for PS/2 port 2");
+			m_devices[1].clear();
+		}
+		
 		if (!m_devices[0] && !m_devices[1])
 			return {};
 

kernel/kernel/Input/PS2/Keyboard.cpp

@@ -1,13 +1,11 @@
 #include <BAN/ScopeGuard.h>
 #include <kernel/CriticalScope.h>
 #include <kernel/FS/DevFS/FileSystem.h>
+#include <kernel/Input/KeyboardLayout.h>
 #include <kernel/Input/PS2/Config.h>
 #include <kernel/Input/PS2/Keyboard.h>
 #include <kernel/Thread.h>
 
-#define SET_MASK(byte, mask, on_off) ((on_off) ? ((byte) | (mask)) : ((byte) & ~(mask)))
-#define TOGGLE_MASK(byte, mask) ((byte) ^ (mask))
-
 namespace Kernel::Input
 {
 
@@ -126,20 +124,24 @@ namespace Kernel::Input
 		if (!keycode.has_value())
 			return;
 
+		auto key = KeyboardLayout::get().get_key_from_event(KeyEvent { .modifier = 0, .keycode = keycode.value() });
+
 		uint16_t modifier_mask = 0;
 		uint16_t toggle_mask = 0;
-		switch (keycode.value())
+		switch (key)
 		{
-			case ModifierKeycode::LShift:	modifier_mask = KeyEvent::Modifier::LShift;	break;
-			case ModifierKeycode::RShift:	modifier_mask = KeyEvent::Modifier::RShift;	break;
-			case ModifierKeycode::LCtrl:	modifier_mask = KeyEvent::Modifier::LCtrl;	break;
-			case ModifierKeycode::RCtrl:	modifier_mask = KeyEvent::Modifier::RCtrl;	break;
-			case ModifierKeycode::LAlt:		modifier_mask = KeyEvent::Modifier::LAlt;	break;
-			case ModifierKeycode::RAlt:		modifier_mask = KeyEvent::Modifier::RAlt;	break;
+			case Key::LeftShift:	modifier_mask = KeyEvent::Modifier::LShift;	break;
+			case Key::RightShift:	modifier_mask = KeyEvent::Modifier::RShift;	break;
+			case Key::LeftCtrl:		modifier_mask = KeyEvent::Modifier::LCtrl;	break;
+			case Key::RightCtrl:	modifier_mask = KeyEvent::Modifier::RCtrl;	break;
+			case Key::LeftAlt:		modifier_mask = KeyEvent::Modifier::LAlt;	break;
+			case Key::RightAlt:		modifier_mask = KeyEvent::Modifier::RAlt;	break;
 
-			case ModifierKeycode::ScrollLock:	toggle_mask = KeyEvent::Modifier::ScrollLock;	break;
-			case ModifierKeycode::NumLock:		toggle_mask = KeyEvent::Modifier::NumLock;		break;
-			case ModifierKeycode::CapsLock:		toggle_mask = KeyEvent::Modifier::CapsLock;		break;
+			case Key::ScrollLock:	toggle_mask = KeyEvent::Modifier::ScrollLock;	break;
+			case Key::NumLock:		toggle_mask = KeyEvent::Modifier::NumLock;		break;
+			case Key::CapsLock:		toggle_mask = KeyEvent::Modifier::CapsLock;		break;
+
+			default: break;
 		}
 
 		if (modifier_mask)
@@ -150,7 +152,7 @@ namespace Kernel::Input
 				m_modifiers |= modifier_mask;
 		}
 
-		if (toggle_mask)
+		if (toggle_mask && !released)
 		{
 			m_modifiers ^= toggle_mask;
 			update_leds();
@@ -173,11 +175,11 @@ namespace Kernel::Input
 	void PS2Keyboard::update_leds()
 	{
 		uint8_t new_leds = 0;
-		if (m_modifiers & +Input::KeyEvent::Modifier::ScrollLock)
+		if (m_modifiers & +KeyEvent::Modifier::ScrollLock)
 			new_leds |= PS2::KBLeds::SCROLL_LOCK;
-		if (m_modifiers & +Input::KeyEvent::Modifier::NumLock)
+		if (m_modifiers & +KeyEvent::Modifier::NumLock)
 			new_leds |= PS2::KBLeds::NUM_LOCK;
-		if (m_modifiers & +Input::KeyEvent::Modifier::CapsLock)
+		if (m_modifiers & +KeyEvent::Modifier::CapsLock)
 			new_leds |= PS2::KBLeds::CAPS_LOCK;
 		append_command_queue(Command::SET_LEDS, new_leds, 0);
 	}

kernel/kernel/PCI.cpp

@@ -1,3 +1,4 @@
+#include <kernel/IDT.h>
 #include <kernel/IO.h>
 #include <kernel/Memory/PageTable.h>
 #include <kernel/MMIO.h>
@@ -5,8 +6,7 @@
 #include <kernel/PCI.h>
 #include <kernel/Storage/ATA/AHCI/Controller.h>
 #include <kernel/Storage/ATA/ATAController.h>
-
-#include <lai/helpers/pci.h>
+#include <kernel/Storage/NVMe/Controller.h>
 
 #define INVALID_VENDOR 0xFFFF
 #define MULTI_FUNCTION 0x80
@@ -25,13 +25,22 @@
 #define PCI_CMD_BUS_MASTER (1 << 2)
 #define PCI_CMD_INTERRUPT_DISABLE (1 << 10)
 
-#define DEBUG_PCI 1
+#define DEBUG_PCI 0
 
 namespace Kernel::PCI
 {
 
 	static PCIManager* s_instance = nullptr;
 
+	struct MSIXEntry
+	{
+		uint32_t msg_addr_low;
+		uint32_t msg_addr_high;
+		uint32_t msg_data;
+		uint32_t vector_ctrl;
+	};
+	static_assert(sizeof(MSIXEntry) == 16);
+
 	uint32_t PCIManager::read_config_dword(uint8_t bus, uint8_t dev, uint8_t func, uint8_t offset)
 	{
 		ASSERT(offset % 4 == 0);
@@ -162,6 +171,14 @@ namespace Kernel::PCI
 							if (auto res = ATAController::create(pci_device); res.is_error())
 								dprintln("ATA: {}", res.error());
 							break;
+						case 0x08:
+							// FIXME: HACK if inode initialization fails before it attaches to DevFS,
+							//        it will kernel panic. This is used to make nvme eternal
+							if (auto res = NVMeController::create(pci_device); res.is_error())
+								dprintln("NVMe: {}", res.error());
+							else
+								res.value()->ref();
+							break;
 						default:
 							dprintln("unsupported storage device (pci {2H}.{2H}.{2H})", pci_device.class_code(), pci_device.subclass(), pci_device.prog_if());
 							break;
@@ -421,38 +438,147 @@ namespace Kernel::PCI
 		}
 	}
 
-	BAN::ErrorOr<uint8_t> PCI::Device::get_irq()
+	BAN::ErrorOr<void> PCI::Device::reserve_irqs(uint8_t count)
 	{
+		if (m_offset_msi_x.has_value())
+		{
+			uint16_t msg_ctrl = read_word(*m_offset_msi_x + 0x02);
+			if (count > (msg_ctrl & 0x7FF) + 1)
+			{
+				dwarnln("MSI-X: could not allocate {} interrupts, only {} supported", count, (msg_ctrl & 0x7FF) + 1);
+				return BAN::Error::from_errno(EFAULT);
+			}
+			msg_ctrl |= 1 << 15; // Enable
+			write_word(*m_offset_msi_x + 0x02, msg_ctrl);
+			disable_pin_interrupts();
+		}
+		else if (m_offset_msi.has_value())
+		{
+			if (count > 1)
+			{
+				dwarnln("MSI: could not allocate {} interrupts, (currently) only {} supported", count, 1);
+				return BAN::Error::from_errno(EFAULT);
+			}
+			uint16_t msg_ctrl = read_word(*m_offset_msi + 0x02);
+			msg_ctrl &= ~(0x07 << 4);	// Only one interrupt
+			msg_ctrl |= 1u << 0;		// Enable
+			write_word(*m_offset_msi + 0x02, msg_ctrl);
+			disable_pin_interrupts();
+		}
+		else if (!InterruptController::get().is_using_apic())
+		{
+			if (count > 1)
+			{
+				dwarnln("PIC: could not allocate {} interrupts, (currently) only {} supported", count, 1);
+				return BAN::Error::from_errno(EFAULT);
+			}
+			enable_pin_interrupts();
+		}
+		else
+		{
+			ASSERT_NOT_REACHED();
+		}
+
+		for (; m_reserved_irq_count < count; m_reserved_irq_count++)
+		{
+			auto irq = InterruptController::get().get_free_irq();
+			if (!irq.has_value())
+			{
+				dwarnln("Could not reserve interrupt for PCI {}:{}.{}", m_bus, m_dev, m_func);
+				return BAN::Error::from_errno(EFAULT);
+			}
+
+			ASSERT(*irq < 32);
+			ASSERT(!(m_reserved_irqs & (1 << *irq)));
+			m_reserved_irqs |= 1 << *irq;
+		}
+
+		return {};
+	}
+
+	static uint64_t msi_message_address()
+	{
+		return 0xFEE00000;
+	}
+
+	static uint32_t msi_message_data(uint8_t irq)
+	{
+		return (IRQ_VECTOR_BASE + irq) & 0xFF;
+	}
+
+	uint8_t PCI::Device::get_irq(uint8_t index)
+	{
+		ASSERT(m_offset_msi.has_value() || m_offset_msi_x.has_value() || !InterruptController::get().is_using_apic());
+		ASSERT(index < m_reserved_irq_count);
+
+		uint8_t count_found = 0;
+		uint8_t irq = 0xFF;
+		for (uint8_t i = 0; i < 32; i++)
+		{
+			if (m_reserved_irqs & (1 << i))
+				count_found++;
+			if (count_found > index)
+			{
+				irq = i;
+				break;
+			}
+		}
+		ASSERT(irq != 0xFF);
+
 		// Legacy PIC just uses the interrupt line field
 		if (!InterruptController::get().is_using_apic())
-			return read_byte(PCI_REG_IRQ_LINE);
-		
-		// TODO: use MSI and MSI-X if supported
-
-		if (m_offset_msi.has_value())
 		{
+			write_byte(PCI_REG_IRQ_LINE, irq);
+			return irq;
 		}
 
 		if (m_offset_msi_x.has_value())
 		{
+			uint32_t dword0 = read_dword(*m_offset_msi_x);
+			ASSERT((dword0 & 0xFF) == 0x11);
+
+			uint32_t dword1 = read_dword(*m_offset_msi_x + 0x04);
+			uint32_t offset = dword1 & ~3u;
+			uint8_t  bir    = dword1 &  3u;
+
+			uint64_t msg_addr = msi_message_address();
+			uint32_t msg_data = msi_message_data(irq);
+
+			auto bar = MUST(allocate_bar_region(bir));
+			ASSERT(bar->type() == BarType::MEM);
+			auto& msi_x_entry = reinterpret_cast<volatile MSIXEntry*>(bar->vaddr() + offset)[index];
+			msi_x_entry.msg_addr_low  = msg_addr & 0xFFFFFFFF;
+			msi_x_entry.msg_addr_high = msg_addr >> 32;;
+			msi_x_entry.msg_data      = msg_data;
+			msi_x_entry.vector_ctrl   = msi_x_entry.vector_ctrl & ~1u;
+
+			return irq;
 		}
 
-		for (uint8_t irq_pin = 1; irq_pin <= 4; irq_pin++)
+		if (m_offset_msi.has_value())
 		{
-			acpi_resource_t dest;
-			auto err = lai_pci_route_pin(&dest, 0, m_bus, m_dev, m_func, irq_pin);
-			if (err != LAI_ERROR_NONE)
+			uint32_t dword0 = read_dword(*m_offset_msi);
+			ASSERT((dword0 & 0xFF) == 0x05);
+
+			uint64_t msg_addr = msi_message_address();
+			uint32_t msg_data = msi_message_data(irq);
+
+			if (dword0 & (1 << 23))
 			{
-				dprintln("{}", lai_api_error_to_string(err));
-				continue;
+				write_dword(*m_offset_msi + 0x04, msg_addr & 0xFFFFFFFF);
+				write_dword(*m_offset_msi + 0x08, msg_addr >> 32);
+				write_word(*m_offset_msi  + 0x12, msg_data);
+			}
+			else
+			{
+				write_dword(*m_offset_msi + 0x04, msg_addr & 0xFFFFFFFF);
+				write_word(*m_offset_msi  + 0x08, msg_data);
 			}
 
-			write_byte(PCI_REG_IRQ_PIN, irq_pin);
-			return dest.base;
+			return irq;
 		}
 
-		dwarnln("Could not allocate interrupt for PCI {}:{}.{}", m_bus, m_dev, m_func);
-		return BAN::Error::from_errno(ENOTSUP);
+		ASSERT_NOT_REACHED();
 	}
 
 	void PCI::Device::set_command_bits(uint16_t mask)

kernel/kernel/PIC.cpp

@@ -1,3 +1,4 @@
+#include <kernel/CriticalScope.h>
 #include <kernel/IDT.h>
 #include <kernel/IO.h>
 #include <kernel/PIC.h>
@@ -70,6 +71,7 @@ namespace Kernel
 
 	void PIC::eoi(uint8_t irq)
 	{
+		ASSERT(!interrupts_enabled());
 		if (irq >= 8)
 			IO::outb(PIC2_CMD, PIC_EOI);
 		IO::outb(PIC1_CMD, PIC_EOI);
@@ -77,6 +79,10 @@ namespace Kernel
 
 	void PIC::enable_irq(uint8_t irq)
 	{
+		CriticalScope _;
+		ASSERT(irq < 16);
+		ASSERT(m_reserved_irqs & (1 << irq));
+
 		uint16_t port = PIC1_DATA;
 		if(irq >= 8)
 		{
@@ -86,6 +92,37 @@ namespace Kernel
 		IO::outb(port, IO::inb(port) & ~(1 << irq));
 	}
 
+	BAN::ErrorOr<void> PIC::reserve_irq(uint8_t irq)
+	{
+		if (irq >= 16)
+		{
+			dwarnln("PIC only supports 16 irqs");
+			return BAN::Error::from_errno(EFAULT);
+		}
+		CriticalScope _;
+		if (m_reserved_irqs & (1 << irq))
+		{
+			dwarnln("irq {} is already reserved", irq);
+			return BAN::Error::from_errno(EFAULT);
+		}
+		m_reserved_irqs |= 1 << irq;
+		return {};
+	}
+
+	BAN::Optional<uint8_t> PIC::get_free_irq()
+	{
+		CriticalScope _;
+		for (int irq = 0; irq < 16; irq++)
+		{
+			if (m_reserved_irqs & (1 << irq))
+				continue;
+			m_reserved_irqs |= 1 << irq;
+			return irq;
+		}
+		
+		return {};
+	}
+
 	bool PIC::is_in_service(uint8_t irq)
 	{
 		uint16_t port = PIC1_CMD;

kernel/kernel/Storage/ATA/AHCI/Controller.cpp

@@ -25,8 +25,8 @@ namespace Kernel
 
 		// Enable interrupts and bus mastering
 		m_pci_device.enable_bus_mastering();
-		m_pci_device.enable_pin_interrupts();
-		set_irq(TRY(m_pci_device.get_irq()));
+		TRY(m_pci_device.reserve_irqs(1));
+		set_irq(m_pci_device.get_irq(0));
 		enable_interrupt();
 		abar_mem.ghc = abar_mem.ghc | SATA_GHC_INTERRUPT_ENABLE;
 

kernel/kernel/Storage/ATA/AHCI/Device.cpp

@@ -8,6 +8,8 @@
 namespace Kernel
 {
 
+	static constexpr uint64_t s_ata_timeout = 1000;
+
 	static void start_cmd(volatile HBAPortMemorySpace* port)
 	{
 		while (port->cmd & HBA_PxCMD_CR)
@@ -100,24 +102,26 @@ namespace Kernel
 		auto slot = find_free_command_slot();
 		ASSERT(slot.has_value());
 
-		auto& command_header = ((HBACommandHeader*)m_dma_region->paddr_to_vaddr(m_port->clb))[slot.value()];
+		volatile auto& command_header = reinterpret_cast<volatile HBACommandHeader*>(m_dma_region->paddr_to_vaddr(m_port->clb))[slot.value()];
 		command_header.cfl = sizeof(FISRegisterH2D) / sizeof(uint32_t);
 		command_header.w = 0;
 		command_header.prdtl = 1;
 
-		auto& command_table = *(HBACommandTable*)m_dma_region->paddr_to_vaddr(command_header.ctba);
-		memset(&command_table, 0x00, sizeof(HBACommandTable));
+		volatile auto& command_table = *reinterpret_cast<volatile HBACommandTable*>(m_dma_region->paddr_to_vaddr(command_header.ctba));
+		memset(const_cast<HBACommandTable*>(&command_table), 0x00, sizeof(HBACommandTable));
 		command_table.prdt_entry[0].dba = m_data_dma_region->paddr();
 		command_table.prdt_entry[0].dbc = 511;
 		command_table.prdt_entry[0].i = 1;
 
-		auto& command = *(FISRegisterH2D*)command_table.cfis;
+		volatile auto& command = *reinterpret_cast<volatile FISRegisterH2D*>(command_table.cfis);
 		command.fis_type = FIS_TYPE_REGISTER_H2D;
 		command.c = 1;
 		command.command = ATA_COMMAND_IDENTIFY;
 
+		uint64_t timeout = SystemTimer::get().ms_since_boot() + s_ata_timeout;
 		while (m_port->tfd & (ATA_STATUS_BSY | ATA_STATUS_DRQ))
-			continue;
+			if (SystemTimer::get().ms_since_boot() >= timeout)
+				return BAN::Error::from_errno(ETIMEDOUT);
 
 		m_port->ci = 1 << slot.value();
 
@@ -152,18 +156,17 @@ namespace Kernel
 
 	BAN::ErrorOr<void> AHCIDevice::block_until_command_completed(uint32_t command_slot)
 	{
-		static constexpr uint64_t total_timeout_ms = 5000;
 		static constexpr uint64_t poll_timeout_ms = 10;
 
 		auto start_time = SystemTimer::get().ms_since_boot();
 
-		while (start_time + poll_timeout_ms < SystemTimer::get().ns_since_boot())
+		while (SystemTimer::get().ms_since_boot() < start_time + poll_timeout_ms)
 			if (!(m_port->ci & (1 << command_slot)))
 				return {};
-		
+
 		// FIXME: This should actually block once semaphores support blocking with timeout.
 		//        This doesn't allow scheduler to go properly idle.
-		while (start_time + total_timeout_ms < SystemTimer::get().ns_since_boot())
+		while (SystemTimer::get().ms_since_boot() < start_time + s_ata_timeout)
 		{
 			Scheduler::get().reschedule();
 			if (!(m_port->ci & (1 << command_slot)))
@@ -216,7 +219,7 @@ namespace Kernel
 		auto slot = find_free_command_slot();
 		ASSERT(slot.has_value());
 		
-		auto& command_header = ((HBACommandHeader*)m_dma_region->paddr_to_vaddr(m_port->clb))[slot.value()];
+		volatile auto& command_header = reinterpret_cast<volatile HBACommandHeader*>(m_dma_region->paddr_to_vaddr(m_port->clb))[slot.value()];
 		command_header.cfl = sizeof(FISRegisterH2D) / sizeof(uint32_t);
 		command_header.prdtl = 1;
 		switch (command)
@@ -231,15 +234,15 @@ namespace Kernel
 				ASSERT_NOT_REACHED();
 		}
 
-		auto& command_table = *(HBACommandTable*)m_dma_region->paddr_to_vaddr(command_header.ctba);
-		memset(&command_table, 0x00, sizeof(HBACommandTable));
+		volatile auto& command_table = *reinterpret_cast<HBACommandTable*>(m_dma_region->paddr_to_vaddr(command_header.ctba));
+		memset(const_cast<HBACommandTable*>(&command_table), 0x00, sizeof(HBACommandTable));
 		command_table.prdt_entry[0].dba = m_data_dma_region->paddr() & 0xFFFFFFFF;
 		command_table.prdt_entry[0].dbau = m_data_dma_region->paddr() >> 32;
 		command_table.prdt_entry[0].dbc = sector_count * sector_size() - 1;
 		command_table.prdt_entry[0].i = 1;
 
-		auto& fis_command = *(FISRegisterH2D*)command_table.cfis;
-		memset(&fis_command, 0x00, sizeof(FISRegisterH2D));
+		volatile auto& fis_command = *reinterpret_cast<volatile FISRegisterH2D*>(command_table.cfis);
+		memset(const_cast<FISRegisterH2D*>(&fis_command), 0x00, sizeof(FISRegisterH2D));
 		fis_command.fis_type = FIS_TYPE_REGISTER_H2D;
 		fis_command.c = 1;
 
@@ -283,8 +286,8 @@ namespace Kernel
 	BAN::Optional<uint32_t> AHCIDevice::find_free_command_slot()
 	{
 		uint32_t slots = m_port->sact | m_port->ci;
-		for (uint32_t i = 0; i < m_controller->command_slot_count(); i++, slots >>= 1)
-			if (!(slots & 1))
+		for (uint32_t i = 0; i < m_controller->command_slot_count(); i++)
+			if (!(slots & (1 << i)))
 				return i;
 		return {};
 	}

kernel/kernel/Storage/ATA/ATABus.cpp

@@ -11,6 +11,8 @@
 namespace Kernel
 {
 
+	static constexpr uint64_t s_ata_timeout_ms = 100;
+
 	BAN::ErrorOr<BAN::RefPtr<ATABus>> ATABus::create(uint16_t base, uint16_t ctrl, uint8_t irq)
 	{
 		auto* bus_ptr = new ATABus(base, ctrl);
@@ -19,8 +21,6 @@ namespace Kernel
 		auto bus = BAN::RefPtr<ATABus>::adopt(bus_ptr);
 		bus->set_irq(irq);
 		TRY(bus->initialize());
-		if (bus->m_devices.empty())
-			return BAN::Error::from_errno(ENODEV);
 		return bus;
 	}
 
@@ -73,16 +73,37 @@ namespace Kernel
 		select_delay();
 	}
 
-	static bool identify_all_ones(BAN::Span<const uint16_t> identify_data)
+	static bool identify_all_same(BAN::Span<const uint16_t> identify_data)
 	{
-		for (size_t i = 0; i < 256; i++)
-			if (identify_data[i] != 0xFFFF)
+		uint16_t value = identify_data[0];
+		for (size_t i = 1; i < 256; i++)
+			if (identify_data[i] != value)
 				return false;
 		return true;
 	}
 
 	BAN::ErrorOr<ATABus::DeviceType> ATABus::identify(bool secondary, BAN::Span<uint16_t> buffer)
 	{
+		// Try to detect whether port contains device
+		uint8_t status = io_read(ATA_PORT_STATUS);
+		if (status & ATA_STATUS_BSY)
+		{
+			uint64_t timeout = SystemTimer::get().ms_since_boot() + s_ata_timeout_ms;
+			while ((status = io_read(ATA_PORT_STATUS)) & ATA_STATUS_BSY)
+			{
+				if (SystemTimer::get().ms_since_boot() >= timeout)
+				{
+					dprintln("BSY flag clear timeout, assuming no drive on port");
+					return BAN::Error::from_errno(ETIMEDOUT);
+				}
+			}
+		}
+		if (__builtin_popcount(status) >= 4)
+		{
+			dprintln("STATUS contains garbage, assuming no drive on port");
+			return BAN::Error::from_errno(EINVAL);
+		}
+
 		select_device(secondary);
 
 		// Disable interrupts
@@ -125,7 +146,7 @@ namespace Kernel
 		ASSERT(buffer.size() >= 256);
 		read_buffer(ATA_PORT_DATA, buffer.data(), 256);
 
-		if (identify_all_ones(buffer))
+		if (identify_all_same(buffer))
 			return BAN::Error::from_errno(ENODEV);
 
 		return type;
@@ -189,12 +210,17 @@ namespace Kernel
 		for (uint32_t i = 0; i < 4; i++)
 			io_read(ATA_PORT_ALT_STATUS);
 
-		uint8_t status = ATA_STATUS_BSY;
-		while (status & ATA_STATUS_BSY)
-			status = io_read(ATA_PORT_STATUS);
+		uint64_t timeout = SystemTimer::get().ms_since_boot() + s_ata_timeout_ms;
+
+		uint8_t status;
+		while ((status = io_read(ATA_PORT_STATUS)) & ATA_STATUS_BSY)
+			if (SystemTimer::get().ms_since_boot() >= timeout)
+				return BAN::Error::from_errno(ETIMEDOUT);
 
 		while (wait_drq && !(status & ATA_STATUS_DRQ))
 		{
+			if (SystemTimer::get().ms_since_boot() >= timeout)
+				return BAN::Error::from_errno(ETIMEDOUT);
 			if (status & ATA_STATUS_ERR)
 				return error();
 			if (status & ATA_STATUS_DF)

kernel/kernel/Storage/ATA/ATAController.cpp

@@ -41,6 +41,8 @@ namespace Kernel
 
 		uint8_t prog_if = m_pci_device.read_byte(0x09);
 
+		// FIXME: support native mode
+
 		if ((prog_if & ATA_PROGIF_CAN_MODIFY_PRIMARY_NATIVE) && (prog_if & ATA_PROGIF_PRIMARY_NATIVE))
 		{
 			prog_if &= ~ATA_PROGIF_PRIMARY_NATIVE;
@@ -57,11 +59,16 @@ namespace Kernel
 
 		if (!(prog_if & ATA_PROGIF_PRIMARY_NATIVE))
 		{
-			auto bus_or_error = ATABus::create(0x1F0, 0x3F6, 14);
-			if (bus_or_error.is_error())
-				dprintln("IDE ATABus: {}", bus_or_error.error());
+			if (InterruptController::get().reserve_irq(14).is_error())
+				dwarnln("Could not reserve interrupt {} for ATA device", 14);
 			else
-				TRY(buses.push_back(bus_or_error.release_value()));
+			{
+				auto bus_or_error = ATABus::create(0x1F0, 0x3F6, 14);
+				if (bus_or_error.is_error())
+					dprintln("IDE ATABus: {}", bus_or_error.error());
+				else
+					TRY(buses.push_back(bus_or_error.release_value()));
+			}
 		}
 		else
 		{
@@ -70,11 +77,16 @@ namespace Kernel
 
 		if (!(prog_if & ATA_PROGIF_SECONDARY_NATIVE))
 		{
-			auto bus_or_error = ATABus::create(0x170, 0x376, 15);
-			if (bus_or_error.is_error())
-				dprintln("IDE ATABus: {}", bus_or_error.error());
+			if (InterruptController::get().reserve_irq(15).is_error())
+				dwarnln("Could not reserver interrupt {} for ATA device", 15);
 			else
-				TRY(buses.push_back(bus_or_error.release_value()));
+			{
+				auto bus_or_error = ATABus::create(0x170, 0x376, 15);
+				if (bus_or_error.is_error())
+					dprintln("IDE ATABus: {}", bus_or_error.error());
+				else
+					TRY(buses.push_back(bus_or_error.release_value()));
+			}
 		}
 		else
 		{

kernel/kernel/Storage/ATA/ATADevice.cpp

@@ -76,7 +76,7 @@ namespace Kernel
 		add_disk_cache();
 
 		DevFileSystem::get().add_device(this);
-		if (auto res = initialize_partitions(); res.is_error())
+		if (auto res = initialize_partitions(name()); res.is_error())
 			dprintln("{}", res.error());
 
 		return {};

kernel/kernel/Storage/NVMe/Controller.cpp

@@ -0,0 +1,310 @@
+#include <BAN/Array.h>
+#include <kernel/FS/DevFS/FileSystem.h>
+#include <kernel/Memory/DMARegion.h>
+#include <kernel/Storage/NVMe/Controller.h>
+#include <kernel/Timer/Timer.h>
+
+#include <sys/sysmacros.h>
+
+#define DEBUG_NVMe 1
+
+namespace Kernel
+{
+
+	static dev_t get_ctrl_dev_major()
+	{
+		static dev_t major = DevFileSystem::get().get_next_dev();
+		return major;
+	}
+
+	static dev_t get_ctrl_dev_minor()
+	{
+		static dev_t minor = 0;
+		return minor++;
+	}
+
+	BAN::ErrorOr<BAN::RefPtr<StorageController>> NVMeController::create(PCI::Device& pci_device)
+	{
+		auto* controller_ptr = new NVMeController(pci_device);
+		if (controller_ptr == nullptr)
+			return BAN::Error::from_errno(ENOMEM);
+		auto controller = BAN::RefPtr<StorageController>::adopt(controller_ptr);
+		TRY(controller->initialize());
+		return controller;
+	}
+
+	NVMeController::NVMeController(PCI::Device& pci_device)
+		: CharacterDevice(0600, 0, 0)
+		, m_pci_device(pci_device)
+		, m_rdev(makedev(get_ctrl_dev_major(), get_ctrl_dev_minor()))
+	{
+		ASSERT(minor(m_rdev) < 10);
+		strcpy(m_name, "nvmeX");
+		m_name[4] = '0' + minor(m_rdev);
+	}
+
+	BAN::ErrorOr<void> NVMeController::initialize()
+	{
+		// See NVM express base specification section 3.5.1
+		m_pci_device.enable_bus_mastering();
+		m_pci_device.enable_memory_space();
+
+		m_bar0 = TRY(m_pci_device.allocate_bar_region(0));
+		if (m_bar0->type() != PCI::BarType::MEM)
+		{
+			dwarnln("NVMe controller BAR0 is not MEM");
+			return BAN::Error::from_errno(EINVAL);
+		}
+		if (m_bar0->size() < 0x1000)
+		{
+			dwarnln("NVMe controller BAR0 is too small {} bytes", m_bar0->size());
+			return BAN::Error::from_errno(EINVAL);	
+		}
+
+		m_controller_registers = reinterpret_cast<volatile NVMe::ControllerRegisters*>(m_bar0->vaddr());
+
+		const auto& vs = m_controller_registers->vs;
+		if (vs.major != 1)
+		{
+			dwarnln("NVMe controller has unsupported version {}.{}", (uint16_t)vs.major, (uint8_t)vs.minor);
+			return BAN::Error::from_errno(ENOTSUP);
+		}
+
+		dprintln_if(DEBUG_NVMe, "NVMe controller");
+		dprintln_if(DEBUG_NVMe, "  version: {}.{}", (uint16_t)vs.major, (uint8_t)vs.minor);
+
+		auto& cap = m_controller_registers->cap;
+		if (!(cap.css & NVMe::CAP_CSS_NVME))
+		{
+			dwarnln("NVMe controller does not support NVMe command set");
+			return BAN::Error::from_errno(ECANCELED);
+		}
+
+		const uint64_t min_page_size = 1ull << (12 + cap.mpsmin);
+		const uint64_t max_page_size = 1ull << (12 + cap.mpsmax);
+		if (PAGE_SIZE < min_page_size || PAGE_SIZE > max_page_size)
+		{
+			dwarnln("NVMe controller does not support {} byte pages, only {}-{} byte pages are supported", PAGE_SIZE, min_page_size, max_page_size);
+			return BAN::Error::from_errno(ECANCELED);
+		}
+
+		// One for aq and one for ioq
+		TRY(m_pci_device.reserve_irqs(2));
+
+		auto& cc = m_controller_registers->cc;
+
+		if (cc.en)
+			TRY(wait_until_ready(true));
+		cc.en = 0;
+		TRY(wait_until_ready(false));
+		dprintln_if(DEBUG_NVMe, " controller reset");
+
+		TRY(create_admin_queue());
+		dprintln_if(DEBUG_NVMe, " created admin queue");
+
+		// Configure controller
+		cc.ams = 0;
+		cc.mps = PAGE_SIZE_SHIFT - 12;
+		cc.css = 0b000;
+
+		cc.en = 1;
+		TRY(wait_until_ready(true));
+		dprintln_if(DEBUG_NVMe, " controller enabled");
+
+		TRY(identify_controller());
+
+		cc.iocqes = 4; static_assert(1 << 4 == sizeof(NVMe::CompletionQueueEntry));
+		cc.iosqes = 6; static_assert(1 << 6 == sizeof(NVMe::SubmissionQueueEntry));
+		TRY(create_io_queue());
+		dprintln_if(DEBUG_NVMe, " created io queue");
+
+		TRY(identify_namespaces());
+
+		DevFileSystem::get().add_device(this);
+
+		return {};
+	}
+
+	BAN::ErrorOr<void> NVMeController::wait_until_ready(bool expected_value)
+	{
+		const auto& cap = m_controller_registers->cap;
+		const auto& csts = m_controller_registers->csts;
+
+		uint64_t timeout = SystemTimer::get().ms_since_boot() + 500 * cap.to;
+		while (csts.rdy != expected_value)
+		{
+			if (SystemTimer::get().ms_since_boot() >= timeout)
+			{
+				dwarnln("NVMe controller reset timedout");
+				return BAN::Error::from_errno(ETIMEDOUT);
+			}
+		}
+
+		return {};
+	}
+
+	BAN::ErrorOr<void> NVMeController::identify_controller()
+	{
+		auto dma_page = TRY(DMARegion::create(PAGE_SIZE));
+
+		NVMe::SubmissionQueueEntry sqe {};
+		sqe.opc = NVMe::OPC_ADMIN_IDENTIFY;
+		sqe.identify.dptr.prp1 = dma_page->paddr();
+		sqe.identify.cns = NVMe::CNS_INDENTIFY_CONTROLLER;
+		if (uint16_t status = m_admin_queue->submit_command(sqe))
+		{
+			dwarnln("NVMe controller identify failed (status {4H})", status);
+			return BAN::Error::from_errno(EFAULT);
+		}
+
+		if (*reinterpret_cast<uint16_t*>(dma_page->vaddr()) != m_pci_device.vendor_id())
+		{
+			dwarnln("NVMe controller vendor id does not match with the one in PCI");
+			return BAN::Error::from_errno(EFAULT);
+		}
+
+		dprintln_if(DEBUG_NVMe, " model: '{}'", BAN::StringView { (char*)dma_page->vaddr() + 24, 20 });
+
+		return {};
+	}
+
+	BAN::ErrorOr<void> NVMeController::identify_namespaces()
+	{
+		auto dma_page = TRY(DMARegion::create(PAGE_SIZE));
+		
+		BAN::Vector<uint32_t> namespace_ids;
+		TRY(namespace_ids.resize(PAGE_SIZE / sizeof(uint32_t)));
+
+		{
+			NVMe::SubmissionQueueEntry sqe {};
+			sqe.opc = NVMe::OPC_ADMIN_IDENTIFY;
+			sqe.identify.dptr.prp1 = dma_page->paddr();
+			sqe.identify.cns = NVMe::CNS_INDENTIFY_ACTIVE_NAMESPACES;
+			if (uint16_t status = m_admin_queue->submit_command(sqe))
+			{
+				dwarnln("NVMe active namespace identify failed (status {4H})", status);
+				return BAN::Error::from_errno(EFAULT);
+			}
+			memcpy(namespace_ids.data(), reinterpret_cast<void*>(dma_page->vaddr()), PAGE_SIZE);
+		}
+
+		for (uint32_t nsid : namespace_ids)
+		{
+			if (nsid == 0)
+				break;
+			dprintln(" found namespace {}", nsid);
+
+			NVMe::SubmissionQueueEntry sqe {};
+			sqe.opc = NVMe::OPC_ADMIN_IDENTIFY;
+			sqe.identify.nsid = nsid;
+			sqe.identify.dptr.prp1 = dma_page->paddr();
+			sqe.identify.cns = NVMe::CNS_INDENTIFY_NAMESPACE;
+			if (uint16_t status = m_admin_queue->submit_command(sqe))
+			{
+				dwarnln("NVMe namespace {} identify failed (status {4H})", nsid , status);
+				return BAN::Error::from_errno(EFAULT);
+			}
+
+			auto& namespace_info = *reinterpret_cast<volatile NVMe::NamespaceIdentify*>(dma_page->vaddr());
+
+			const uint64_t block_count = namespace_info.nsze;
+
+			const uint64_t format = namespace_info.lbafN[namespace_info.flbas & 0x0F];
+			const uint64_t block_size = 1u << ((format >> 16) & 0xFF);
+
+			dprintln("   block count {}",     block_count);
+			dprintln("   block size  {} B",   block_size);
+			dprintln("   total       {} MiB", block_count * block_size / (1 << 20));
+
+			auto ns = TRY(NVMeNamespace::create(*this, nsid, block_count, block_size));
+			TRY(m_namespaces.push_back(BAN::move(ns)));
+		}
+
+		return {};
+	}
+
+	BAN::ErrorOr<void> NVMeController::create_admin_queue()
+	{
+		const uint32_t admin_queue_depth = BAN::Math::min(PAGE_SIZE / sizeof(NVMe::CompletionQueueEntry), PAGE_SIZE / sizeof(NVMe::SubmissionQueueEntry));
+		auto& aqa = m_controller_registers->aqa;
+		aqa.acqs = admin_queue_depth - 1;
+		aqa.asqs = admin_queue_depth - 1;
+		dprintln_if(DEBUG_NVMe, " admin queue depth is {}", admin_queue_depth);
+
+		const uint32_t completion_queue_size = admin_queue_depth * sizeof(NVMe::CompletionQueueEntry);
+		auto completion_queue = TRY(DMARegion::create(completion_queue_size));
+		memset((void*)completion_queue->vaddr(), 0x00, completion_queue->size());
+
+		const uint32_t submission_queue_size = admin_queue_depth * sizeof(NVMe::SubmissionQueueEntry);
+		auto submission_queue = TRY(DMARegion::create(submission_queue_size));
+		memset((void*)submission_queue->vaddr(), 0x00, submission_queue->size());
+
+		m_controller_registers->acq = completion_queue->paddr();
+		m_controller_registers->asq = submission_queue->paddr();
+
+		uint8_t irq = m_pci_device.get_irq(0);
+		dprintln_if(DEBUG_NVMe, " admin queue using irq {}", irq);
+
+		auto& doorbell = *reinterpret_cast<volatile NVMe::DoorbellRegisters*>(m_bar0->vaddr() + NVMe::ControllerRegisters::SQ0TDBL);
+
+		m_admin_queue = TRY(BAN::UniqPtr<NVMeQueue>::create(BAN::move(completion_queue), BAN::move(submission_queue), doorbell, admin_queue_depth, irq));
+
+		return {};
+	}
+
+	BAN::ErrorOr<void> NVMeController::create_io_queue()
+	{
+		constexpr uint32_t queue_size = PAGE_SIZE;
+		constexpr uint32_t queue_elems = queue_size / BAN::Math::max(sizeof(NVMe::CompletionQueueEntry), sizeof(NVMe::SubmissionQueueEntry));
+		auto completion_queue = TRY(DMARegion::create(queue_size));
+		memset((void*)completion_queue->vaddr(), 0x00, completion_queue->size());
+
+		auto submission_queue = TRY(DMARegion::create(queue_size));
+		memset((void*)submission_queue->vaddr(), 0x00, submission_queue->size());
+
+		{
+			NVMe::SubmissionQueueEntry sqe {};
+			sqe.opc = NVMe::OPC_ADMIN_CREATE_CQ;
+			sqe.create_cq.dptr.prp1 = completion_queue->paddr();
+			sqe.create_cq.qsize = queue_elems - 1;
+			sqe.create_cq.qid = 1;
+			sqe.create_cq.iv = 1;
+			sqe.create_cq.ien = 1;
+			sqe.create_cq.pc = 1;
+			if (uint16_t status = m_admin_queue->submit_command(sqe))
+			{
+				dwarnln("NVMe io completion queue creation failed (status {4H})", status);
+				return BAN::Error::from_errno(EFAULT);
+			}
+		}
+
+		{
+			NVMe::SubmissionQueueEntry sqe {};
+			sqe.opc = NVMe::OPC_ADMIN_CREATE_SQ;
+			sqe.create_sq.dptr.prp1 = submission_queue->paddr();
+			sqe.create_sq.qsize = queue_elems - 1;
+			sqe.create_sq.qid = 1;
+			sqe.create_sq.cqid = 1;
+			sqe.create_sq.qprio = 0;
+			sqe.create_sq.pc = 1;
+			sqe.create_sq.nvmsetid = 0;
+			if (uint16_t status = m_admin_queue->submit_command(sqe))
+			{
+				dwarnln("NVMe io submission queue creation failed (status {4H})", status);
+				return BAN::Error::from_errno(EFAULT);
+			}
+		}
+
+		uint8_t irq = m_pci_device.get_irq(1);
+		dprintln_if(DEBUG_NVMe, " io queue using irq {}", irq);
+
+		const uint32_t doorbell_stride = 1 << (2 + m_controller_registers->cap.dstrd);
+		const uint32_t doorbell_offset = 2 * doorbell_stride;
+		auto& doorbell = *reinterpret_cast<volatile NVMe::DoorbellRegisters*>(m_bar0->vaddr() + NVMe::ControllerRegisters::SQ0TDBL + doorbell_offset);
+
+		m_io_queue = TRY(BAN::UniqPtr<NVMeQueue>::create(BAN::move(completion_queue), BAN::move(submission_queue), doorbell, queue_elems, irq));
+
+		return {};
+	}
+
+}

kernel/kernel/Storage/NVMe/Namespace.cpp

@@ -0,0 +1,119 @@
+#include <kernel/FS/DevFS/FileSystem.h>
+#include <kernel/Storage/NVMe/Controller.h>
+#include <kernel/Storage/NVMe/Namespace.h>
+
+#include <sys/sysmacros.h>
+
+namespace Kernel
+{
+
+	static dev_t get_ns_dev_major()
+	{
+		static dev_t major = DevFileSystem::get().get_next_dev();
+		return major;
+	}
+
+	static dev_t get_ns_dev_minor()
+	{
+		static dev_t minor = 0;
+		return minor++;
+	}
+
+	BAN::ErrorOr<BAN::RefPtr<NVMeNamespace>> NVMeNamespace::create(NVMeController& controller, uint32_t nsid, uint64_t block_count, uint32_t block_size)
+	{
+		auto* namespace_ptr = new NVMeNamespace(controller, nsid, block_count, block_size);
+		if (namespace_ptr == nullptr)
+			return BAN::Error::from_errno(ENOMEM);
+		auto ns = BAN::RefPtr<NVMeNamespace>::adopt(namespace_ptr);
+		TRY(ns->initialize());
+		return ns;
+	}
+
+	NVMeNamespace::NVMeNamespace(NVMeController& controller, uint32_t nsid, uint64_t block_count, uint32_t block_size)
+		: m_controller(controller)
+		, m_nsid(nsid)
+		, m_block_size(block_size)
+		, m_block_count(block_count)
+		, m_rdev(makedev(get_ns_dev_major(), get_ns_dev_minor()))
+	{
+		ASSERT(minor(m_rdev) < 10);
+		ASSERT(m_controller.name().size() + 2 < sizeof(m_name));
+		memcpy(m_name, m_controller.name().data(), m_controller.name().size());
+		m_name[m_controller.name().size() + 0] = 'n';
+		m_name[m_controller.name().size() + 1] = '1' + minor(m_rdev);
+		m_name[m_controller.name().size() + 2] = '\0';
+	}
+
+	BAN::ErrorOr<void> NVMeNamespace::initialize()
+	{
+		m_dma_region = TRY(DMARegion::create(PAGE_SIZE));
+
+		add_disk_cache();
+
+		DevFileSystem::get().add_device(this);
+
+		char name_prefix[20];
+		strcpy(name_prefix, m_name);
+		strcat(name_prefix, "p");
+		if (auto res = initialize_partitions(name_prefix); res.is_error())
+			dprintln("{}", res.error());
+
+		return {};
+	}
+
+	BAN::ErrorOr<void> NVMeNamespace::read_sectors_impl(uint64_t lba, uint64_t sector_count, BAN::ByteSpan buffer)
+	{
+		ASSERT(buffer.size() >= sector_count * m_block_size);
+
+		for (uint64_t i = 0; i < sector_count;)
+		{
+			uint16_t count = BAN::Math::min(sector_count - i, m_dma_region->size() / m_block_size);
+
+			NVMe::SubmissionQueueEntry sqe {};
+			sqe.opc = NVMe::OPC_IO_READ;
+			sqe.read.nsid = m_nsid;
+			sqe.read.dptr.prp1 = m_dma_region->paddr();
+			sqe.read.slba = lba + i;
+			sqe.read.nlb = count - 1;
+			if (uint16_t status = m_controller.io_queue().submit_command(sqe))
+			{
+				dwarnln("NVMe read failed (status {4H})", status);
+				return BAN::Error::from_errno(EIO);
+			}
+			memcpy(buffer.data() + i * m_block_size, reinterpret_cast<void*>(m_dma_region->vaddr()), count * m_block_size);
+
+			i += count;
+		}
+
+		return {};
+	}
+
+	BAN::ErrorOr<void> NVMeNamespace::write_sectors_impl(uint64_t lba, uint64_t sector_count, BAN::ConstByteSpan buffer)
+	{
+		ASSERT(buffer.size() >= sector_count * m_block_size);
+
+		for (uint64_t i = 0; i < sector_count;)
+		{
+			uint16_t count = BAN::Math::min(sector_count - i, m_dma_region->size() / m_block_size);
+
+			memcpy(reinterpret_cast<void*>(m_dma_region->vaddr()), buffer.data() + i * m_block_size, count * m_block_size);
+
+			NVMe::SubmissionQueueEntry sqe {};
+			sqe.opc = NVMe::OPC_IO_WRITE;
+			sqe.read.nsid = m_nsid;
+			sqe.read.dptr.prp1 = m_dma_region->paddr();
+			sqe.read.slba = lba + i;
+			sqe.read.nlb = count - 1;
+			if (uint16_t status = m_controller.io_queue().submit_command(sqe))
+			{
+				dwarnln("NVMe write failed (status {4H})", status);
+				return BAN::Error::from_errno(EIO);
+			}
+
+			i += count;
+		}
+
+		return {};
+	}
+
+}

kernel/kernel/Storage/NVMe/Queue.cpp

@@ -0,0 +1,82 @@
+#include <kernel/LockGuard.h>
+#include <kernel/Scheduler.h>
+#include <kernel/Storage/NVMe/Queue.h>
+#include <kernel/Timer/Timer.h>
+
+namespace Kernel
+{
+
+	static constexpr uint64_t s_nvme_command_timeout_ms = 1000;
+	static constexpr uint64_t s_nvme_command_poll_timeout_ms = 20;
+
+	NVMeQueue::NVMeQueue(BAN::UniqPtr<Kernel::DMARegion>&& cq, BAN::UniqPtr<Kernel::DMARegion>&& sq, volatile NVMe::DoorbellRegisters& db, uint32_t qdepth, uint8_t irq)
+		: m_completion_queue(BAN::move(cq))
+		, m_submission_queue(BAN::move(sq))
+		, m_doorbell(db)
+		, m_qdepth(qdepth)
+	{
+		set_irq(irq);
+		enable_interrupt();
+	}
+
+	void NVMeQueue::handle_irq()
+	{
+		auto* cq_ptr = reinterpret_cast<NVMe::CompletionQueueEntry*>(m_completion_queue->vaddr());
+
+		while ((cq_ptr[m_cq_head].sts & 1) == m_cq_valid_phase)
+		{
+			uint16_t sts = cq_ptr[m_cq_head].sts >> 1;
+			uint16_t cid = cq_ptr[m_cq_head].cid;
+			ASSERT(cid == 0);
+
+			ASSERT(!m_done);
+			m_status = sts;
+			m_done = true;
+			m_semaphore.unblock();
+
+			m_cq_head = (m_cq_head + 1) % m_qdepth;
+			if (m_cq_head == 0)
+				m_cq_valid_phase ^= 1;
+		}
+
+		m_doorbell.cq_head = m_cq_head;
+	}
+
+	uint16_t NVMeQueue::submit_command(NVMe::SubmissionQueueEntry& sqe)
+	{
+		LockGuard _(m_lock);
+
+		ASSERT(m_done == false);
+		m_status = 0;
+
+		sqe.cid = 0;
+
+		auto* sqe_ptr = reinterpret_cast<NVMe::SubmissionQueueEntry*>(m_submission_queue->vaddr());
+		memcpy(&sqe_ptr[m_sq_tail], &sqe, sizeof(NVMe::SubmissionQueueEntry));
+		m_sq_tail = (m_sq_tail + 1) % m_qdepth;
+		m_doorbell.sq_tail = m_sq_tail;
+
+		const uint64_t start_time = SystemTimer::get().ms_since_boot();
+		while (SystemTimer::get().ms_since_boot() < start_time + s_nvme_command_poll_timeout_ms)
+		{
+			if (!m_done)
+				continue;
+			m_done = false;
+			return m_status;
+		}
+		
+		while (SystemTimer::get().ms_since_boot() < start_time + s_nvme_command_timeout_ms)
+		{
+			if (!m_done)
+			{
+				m_semaphore.block();
+				continue;
+			}
+			m_done = false;
+			return m_status;
+		}
+
+		return 0xFFFF;
+	}
+
+}

kernel/kernel/Storage/Partition.cpp

@@ -5,15 +5,15 @@
 namespace Kernel
 {
 
-	BAN::ErrorOr<BAN::RefPtr<Partition>> Partition::create(BAN::RefPtr<BlockDevice> device, const BAN::GUID& type, const BAN::GUID& guid, uint64_t first_block, uint64_t last_block, uint64_t attr, const char* label, uint32_t index)
+	BAN::ErrorOr<BAN::RefPtr<Partition>> Partition::create(BAN::RefPtr<BlockDevice> device, const BAN::GUID& type, const BAN::GUID& guid, uint64_t first_block, uint64_t last_block, uint64_t attr, const char* label, uint32_t index, BAN::StringView name_prefix)
 	{
-		auto partition_ptr = new Partition(device, type, guid, first_block, last_block, attr, label, index);
+		auto partition_ptr = new Partition(device, type, guid, first_block, last_block, attr, label, index, name_prefix);
 		if (partition_ptr == nullptr)
 			return BAN::Error::from_errno(ENOMEM);
 		return BAN::RefPtr<Partition>::adopt(partition_ptr);
 	}	
 
-	Partition::Partition(BAN::RefPtr<BlockDevice> device, const BAN::GUID& type, const BAN::GUID& guid, uint64_t first_block, uint64_t last_block, uint64_t attr, const char* label, uint32_t index)
+	Partition::Partition(BAN::RefPtr<BlockDevice> device, const BAN::GUID& type, const BAN::GUID& guid, uint64_t first_block, uint64_t last_block, uint64_t attr, const char* label, uint32_t index, BAN::StringView name_prefix)
 		: BlockDevice(0660, 0, 0)
 		, m_device(device)
 		, m_type(type)
@@ -21,7 +21,7 @@ namespace Kernel
 		, m_first_block(first_block)
 		, m_last_block(last_block)
 		, m_attributes(attr)
-		, m_name(BAN::String::formatted("{}{}", device->name(), index))
+		, m_name(BAN::String::formatted("{}{}", name_prefix, index))
 		, m_rdev(makedev(major(device->rdev()), index))
 	{
 		memcpy(m_label, label, sizeof(m_label));

kernel/kernel/Storage/StorageDevice.cpp

@@ -142,7 +142,7 @@ namespace Kernel
 		return true;
 	}
 
-	BAN::ErrorOr<void> StorageDevice::initialize_partitions()
+	BAN::ErrorOr<void> StorageDevice::initialize_partitions(BAN::StringView name_prefix)
 	{
 		if (total_size() < sizeof(GPTHeader))
 			return BAN::Error::from_error_code(ErrorCode::Storage_GPTHeader);
@@ -189,7 +189,8 @@ namespace Kernel
 				entry.ending_lba,
 				entry.attributes,
 				utf8_name,
-				i + 1
+				i + 1,
+				name_prefix
 			));
 			TRY(m_partitions.push_back(BAN::move(partition)));
 		}

kernel/kernel/Terminal/Serial.cpp

@@ -195,12 +195,14 @@ namespace Kernel
 		if (serial.port() == COM1_PORT)
 		{
 			IO::outb(COM1_PORT + 1, 1);
+			TRY(InterruptController::get().reserve_irq(COM1_IRQ));
 			tty->set_irq(COM1_IRQ);
 			tty->enable_interrupt();
 		}
 		else if (serial.port() == COM2_PORT)
 		{
 			IO::outb(COM2_PORT + 1, 1);
+			TRY(InterruptController::get().reserve_irq(COM2_IRQ));
 			tty->set_irq(COM2_IRQ);
 			tty->enable_interrupt();
 		}

kernel/kernel/Timer/HPET.cpp

@@ -105,6 +105,7 @@ namespace Kernel
 				if (irq_cap & (1 << irq))
 					break;
 		}
+		TRY(InterruptController::get().reserve_irq(irq));
 
 		unmapper.disable();
 

kernel/kernel/kernel.cpp

@@ -182,17 +182,19 @@ static void init2(void*)
 	SystemTimer::get().sleep(5000);
 #endif
 
+	// Initialize empty keymap
+	MUST(Input::KeyboardLayout::initialize());
+	if (auto res = PS2Controller::initialize(); res.is_error())
+		dprintln("{}", res.error());
+
+	// NOTE: PCI devices are the last ones to be initialized
+	//       so other devices can reserve predefined interrupts
 	PCI::PCIManager::initialize();
 	dprintln("PCI initialized");
 
 	VirtualFileSystem::initialize(cmdline.root);
 	dprintln("VFS initialized");
 
-	// Initialize empty keymap
-	MUST(Input::KeyboardLayout::initialize());
-	if (auto res = PS2Controller::initialize(); res.is_error())
-		dprintln("{}", res.error());
-
 	TTY::initialize_devices();
 
 	MUST(Process::create_userspace({ 0, 0, 0, 0 }, "/usr/bin/init"sv));

script/build.sh

@@ -100,6 +100,7 @@ case $1 in
 		build_target clean
 		rm -f $FAKEROOT_FILE
 		rm -rf $BANAN_SYSROOT
+		rm -f $BANAN_DISK_IMAGE_PATH
 		;;
 	*)
 		build_target $1

script/qemu.sh

@@ -13,11 +13,16 @@ if (($BANAN_UEFI_BOOT)); then
 	BIOS_ARGS="-bios $OVMF_PATH -net none"
 fi
 
+if [[ $BANAN_DISK_TYPE == "NVME" ]]; then
+	DISK_ARGS="-device nvme,serial=deadbeef,drive=disk"
+else
+	DISK_ARGS="-device ahci,id=ahci -device ide-hd,drive=disk,bus=ahci.0"
+fi
+
 qemu-system-$BANAN_ARCH												\
 	-m 128															\
 	-smp 2															\
 	$BIOS_ARGS														\
 	-drive format=raw,id=disk,file=${BANAN_DISK_IMAGE_PATH},if=none	\
-	-device ahci,id=ahci											\
-	-device ide-hd,drive=disk,bus=ahci.0							\
+	$DISK_ARGS														\
 	$@																\