Kernel: Implement MSI, MSI-X and interrupt reservation

kernel/include/kernel/APIC.h

@@ -14,6 +14,9 @@ namespace Kernel
 		virtual void enable_irq(uint8_t) override;
 		virtual bool is_in_service(uint8_t) override;
 
+		virtual BAN::ErrorOr<void> reserve_irq(uint8_t irq) override;
+		virtual BAN::Optional<uint8_t> get_free_irq() override;
+
 	private:
 		uint32_t read_from_local_apic(ptrdiff_t);
 		void write_to_local_apic(ptrdiff_t, uint32_t);
@@ -54,6 +57,7 @@ namespace Kernel
 		Kernel::vaddr_t			m_local_apic_vaddr = 0;
 		BAN::Vector<IOAPIC>		m_io_apics;	
 		uint8_t					m_irq_overrides[0x100] {};
+		uint8_t					m_reserved_gsis[0x100 / 8] {};
 	};
 
 }

kernel/include/kernel/InterruptController.h

@@ -1,5 +1,8 @@
 #pragma once
 
+#include <BAN/Optional.h>
+#include <BAN/Errors.h>
+
 #include <stdint.h>
 
 #define DISABLE_INTERRUPTS() asm volatile("cli")
@@ -37,6 +40,9 @@ namespace Kernel
 		static void initialize(bool force_pic);
 		static InterruptController& get();
 
+		virtual BAN::ErrorOr<void> reserve_irq(uint8_t irq) = 0;
+		virtual BAN::Optional<uint8_t> get_free_irq() = 0;
+
 		bool is_using_apic() const { return m_using_apic; }
 
 		void enter_acpi_mode();

kernel/include/kernel/PCI.h

@@ -79,7 +79,8 @@ namespace Kernel::PCI
 		uint16_t vendor_id() const { return m_vendor_id; }
 		uint16_t device_id() const { return m_device_id; }
 
-		BAN::ErrorOr<uint8_t> get_irq();
+		BAN::ErrorOr<void> reserve_irqs(uint8_t count);
+		uint8_t get_irq(uint8_t index);
 
 		BAN::ErrorOr<BAN::UniqPtr<BarRegion>> allocate_bar_region(uint8_t bar_num);
 
@@ -92,15 +93,15 @@ namespace Kernel::PCI
 		void enable_io_space();
 		void disable_io_space();
 
-		void enable_pin_interrupts();
-		void disable_pin_interrupts();
-
 	private:
 		void enumerate_capabilites();
 
 		void set_command_bits(uint16_t mask);
 		void unset_command_bits(uint16_t mask);
 
+		void enable_pin_interrupts();
+		void disable_pin_interrupts();
+
 	private:
 		const uint8_t m_bus;
 		const uint8_t m_dev;
@@ -114,6 +115,9 @@ namespace Kernel::PCI
 		uint16_t m_vendor_id;
 		uint16_t m_device_id;
 
+		uint32_t m_reserved_irqs { 0 };
+		uint8_t m_reserved_irq_count { 0 };
+
 		BAN::Optional<uint8_t> m_offset_msi;
 		BAN::Optional<uint8_t> m_offset_msi_x;
 	};

kernel/include/kernel/PIC.h

@@ -12,12 +12,16 @@ namespace Kernel
 		virtual void enable_irq(uint8_t) override;
 		virtual bool is_in_service(uint8_t) override;
 
+		virtual BAN::ErrorOr<void> reserve_irq(uint8_t irq) override;
+		virtual BAN::Optional<uint8_t> get_free_irq() override;
+
 		static void remap();
 		static void mask_all();
 
 	private:
 		static PIC* create();
 		friend class InterruptController;
+		uint16_t m_reserved_irqs { 0 };
 	};
 
 }

kernel/kernel/APIC.cpp

@@ -223,8 +223,16 @@ namespace Kernel
 
 	void APIC::enable_irq(uint8_t irq)
 	{
+		CriticalScope _;
+
 		uint32_t gsi = m_irq_overrides[irq];
 
+		{
+			int byte = gsi / 8;
+			int bit  = gsi % 8;
+			ASSERT(m_reserved_gsis[byte] & (1 << bit));
+		}
+
 		IOAPIC* ioapic = nullptr;
 		for (IOAPIC& io : m_io_apics)
 		{
@@ -258,4 +266,67 @@ namespace Kernel
 		return isr & (1 << bit);
 	}
 
+	BAN::ErrorOr<void> APIC::reserve_irq(uint8_t irq)
+	{
+		CriticalScope _;
+
+		uint32_t gsi = m_irq_overrides[irq];
+
+		IOAPIC* ioapic = nullptr;
+		for (IOAPIC& io : m_io_apics)
+		{
+			if (io.gsi_base <= gsi && gsi <= io.gsi_base + io.max_redirs)
+			{
+				ioapic = &io;
+				break;
+			}
+		}
+
+		if (!ioapic)
+		{
+			dwarnln("Cannot enable irq {} for APIC", irq);
+			return BAN::Error::from_errno(EFAULT);
+		}
+
+		int byte = gsi / 8;
+		int bit  = gsi % 8;
+		if (m_reserved_gsis[byte] & (1 << bit))
+		{
+			dwarnln("irq {} is already reserved", irq);
+			return BAN::Error::from_errno(EFAULT);
+		}
+		m_reserved_gsis[byte] |= 1 << bit;
+		return {};
+	}
+
+	BAN::Optional<uint8_t> APIC::get_free_irq()
+	{
+		CriticalScope _;
+		for (int irq = 0; irq <= 0xFF; irq++)
+		{
+			uint32_t gsi = m_irq_overrides[irq];
+		
+			IOAPIC* ioapic = nullptr;
+			for (IOAPIC& io : m_io_apics)
+			{
+				if (io.gsi_base <= gsi && gsi <= io.gsi_base + io.max_redirs)
+				{
+					ioapic = &io;
+					break;
+				}
+			}
+
+			if (!ioapic)
+				continue;
+
+			int byte = gsi / 8;
+			int bit  = gsi % 8;
+			if (m_reserved_gsis[byte] & (1 << bit))
+				continue;
+			m_reserved_gsis[byte] |= 1 << bit;
+			return irq;
+		}
+		return {};
+	}
+
 }

kernel/kernel/Input/PS2/Controller.cpp

@@ -329,6 +329,18 @@ namespace Kernel::Input
 			}
 		}
 
+		// Reserve IRQs
+		if (m_devices[0] && InterruptController::get().reserve_irq(PS2::IRQ::DEVICE0).is_error())
+		{
+			dwarnln("Could not reserve irq for PS/2 port 1");
+			m_devices[0].clear();
+		}
+		if (m_devices[1] && InterruptController::get().reserve_irq(PS2::IRQ::DEVICE1).is_error())
+		{
+			dwarnln("Could not reserve irq for PS/2 port 2");
+			m_devices[1].clear();
+		}
+		
 		if (!m_devices[0] && !m_devices[1])
 			return {};
 

kernel/kernel/PCI.cpp

@@ -1,3 +1,4 @@
+#include <kernel/IDT.h>
 #include <kernel/IO.h>
 #include <kernel/Memory/PageTable.h>
 #include <kernel/MMIO.h>
@@ -6,8 +7,6 @@
 #include <kernel/Storage/ATA/AHCI/Controller.h>
 #include <kernel/Storage/ATA/ATAController.h>
 
-#include <lai/helpers/pci.h>
-
 #define INVALID_VENDOR 0xFFFF
 #define MULTI_FUNCTION 0x80
 
@@ -25,13 +24,22 @@
 #define PCI_CMD_BUS_MASTER (1 << 2)
 #define PCI_CMD_INTERRUPT_DISABLE (1 << 10)
 
-#define DEBUG_PCI 1
+#define DEBUG_PCI 0
 
 namespace Kernel::PCI
 {
 
 	static PCIManager* s_instance = nullptr;
 
+	struct MSIXEntry
+	{
+		uint32_t msg_addr_low;
+		uint32_t msg_addr_high;
+		uint32_t msg_data;
+		uint32_t vector_ctrl;
+	};
+	static_assert(sizeof(MSIXEntry) == 16);
+
 	uint32_t PCIManager::read_config_dword(uint8_t bus, uint8_t dev, uint8_t func, uint8_t offset)
 	{
 		ASSERT(offset % 4 == 0);
@@ -421,38 +429,147 @@ namespace Kernel::PCI
 		}
 	}
 
-	BAN::ErrorOr<uint8_t> PCI::Device::get_irq()
+	BAN::ErrorOr<void> PCI::Device::reserve_irqs(uint8_t count)
 	{
+		if (m_offset_msi_x.has_value())
+		{
+			uint16_t msg_ctrl = read_word(*m_offset_msi_x + 0x02);
+			if (count > (msg_ctrl & 0x7FF) + 1)
+			{
+				dwarnln("MSI-X: could not allocate {} interrupts, only {} supported", count, (msg_ctrl & 0x7FF) + 1);
+				return BAN::Error::from_errno(EFAULT);
+			}
+			msg_ctrl |= 1 << 15; // Enable
+			write_word(*m_offset_msi_x + 0x02, msg_ctrl);
+			disable_pin_interrupts();
+		}
+		else if (m_offset_msi.has_value())
+		{
+			if (count > 1)
+			{
+				dwarnln("MSI: could not allocate {} interrupts, (currently) only {} supported", count, 1);
+				return BAN::Error::from_errno(EFAULT);
+			}
+			uint16_t msg_ctrl = read_word(*m_offset_msi + 0x02);
+			msg_ctrl &= ~(0x07 << 4);	// Only one interrupt
+			msg_ctrl |= 1u << 0;		// Enable
+			write_word(*m_offset_msi + 0x02, msg_ctrl);
+			disable_pin_interrupts();
+		}
+		else if (!InterruptController::get().is_using_apic())
+		{
+			if (count > 1)
+			{
+				dwarnln("PIC: could not allocate {} interrupts, (currently) only {} supported", count, 1);
+				return BAN::Error::from_errno(EFAULT);
+			}
+			enable_pin_interrupts();
+		}
+		else
+		{
+			ASSERT_NOT_REACHED();
+		}
+
+		for (; m_reserved_irq_count < count; m_reserved_irq_count++)
+		{
+			auto irq = InterruptController::get().get_free_irq();
+			if (!irq.has_value())
+			{
+				dwarnln("Could not reserve interrupt for PCI {}:{}.{}", m_bus, m_dev, m_func);
+				return BAN::Error::from_errno(EFAULT);
+			}
+
+			ASSERT(*irq < 32);
+			ASSERT(!(m_reserved_irqs & (1 << *irq)));
+			m_reserved_irqs |= 1 << *irq;
+		}
+
+		return {};
+	}
+
+	static uint64_t msi_message_address()
+	{
+		return 0xFEE00000;
+	}
+
+	static uint32_t msi_message_data(uint8_t irq)
+	{
+		return (IRQ_VECTOR_BASE + irq) & 0xFF;
+	}
+
+	uint8_t PCI::Device::get_irq(uint8_t index)
+	{
+		ASSERT(m_offset_msi.has_value() || m_offset_msi_x.has_value() || !InterruptController::get().is_using_apic());
+		ASSERT(index < m_reserved_irq_count);
+
+		uint8_t count_found = 0;
+		uint8_t irq = 0xFF;
+		for (uint8_t i = 0; i < 32; i++)
+		{
+			if (m_reserved_irqs & (1 << i))
+				count_found++;
+			if (count_found > index)
+			{
+				irq = i;
+				break;
+			}
+		}
+		ASSERT(irq != 0xFF);
+
 		// Legacy PIC just uses the interrupt line field
 		if (!InterruptController::get().is_using_apic())
-			return read_byte(PCI_REG_IRQ_LINE);
-		
-		// TODO: use MSI and MSI-X if supported
-
-		if (m_offset_msi.has_value())
 		{
+			write_byte(PCI_REG_IRQ_LINE, irq);
+			return irq;
 		}
 
 		if (m_offset_msi_x.has_value())
 		{
+			uint32_t dword0 = read_dword(*m_offset_msi_x);
+			ASSERT((dword0 & 0xFF) == 0x11);
+
+			uint32_t dword1 = read_dword(*m_offset_msi_x + 0x04);
+			uint32_t offset = dword1 & ~3u;
+			uint8_t  bir    = dword1 &  3u;
+
+			uint64_t msg_addr = msi_message_address();
+			uint32_t msg_data = msi_message_data(irq);
+
+			auto bar = MUST(allocate_bar_region(bir));
+			ASSERT(bar->type() == BarType::MEM);
+			auto& msi_x_entry = reinterpret_cast<volatile MSIXEntry*>(bar->vaddr() + offset)[index];
+			msi_x_entry.msg_addr_low  = msg_addr & 0xFFFFFFFF;
+			msi_x_entry.msg_addr_high = msg_addr >> 32;;
+			msi_x_entry.msg_data      = msg_data;
+			msi_x_entry.vector_ctrl   = msi_x_entry.vector_ctrl & ~1u;
+
+			return irq;
 		}
 
-		for (uint8_t irq_pin = 1; irq_pin <= 4; irq_pin++)
+		if (m_offset_msi.has_value())
 		{
-			acpi_resource_t dest;
+			uint32_t dword0 = read_dword(*m_offset_msi);
-			auto err = lai_pci_route_pin(&dest, 0, m_bus, m_dev, m_func, irq_pin);
+			ASSERT((dword0 & 0xFF) == 0x05);
-			if (err != LAI_ERROR_NONE)
+
+			uint64_t msg_addr = msi_message_address();
+			uint32_t msg_data = msi_message_data(irq);
+
+			if (dword0 & (1 << 23))
 			{
-				dprintln("{}", lai_api_error_to_string(err));
+				write_dword(*m_offset_msi + 0x04, msg_addr & 0xFFFFFFFF);
-				continue;
+				write_dword(*m_offset_msi + 0x08, msg_addr >> 32);
+				write_word(*m_offset_msi  + 0x12, msg_data);
+			}
+			else
+			{
+				write_dword(*m_offset_msi + 0x04, msg_addr & 0xFFFFFFFF);
+				write_word(*m_offset_msi  + 0x08, msg_data);
 			}
 
-			write_byte(PCI_REG_IRQ_PIN, irq_pin);
+			return irq;
-			return dest.base;
 		}
 
-		dwarnln("Could not allocate interrupt for PCI {}:{}.{}", m_bus, m_dev, m_func);
+		ASSERT_NOT_REACHED();
-		return BAN::Error::from_errno(ENOTSUP);
 	}
 
 	void PCI::Device::set_command_bits(uint16_t mask)

kernel/kernel/PIC.cpp

@@ -1,3 +1,4 @@
+#include <kernel/CriticalScope.h>
 #include <kernel/IDT.h>
 #include <kernel/IO.h>
 #include <kernel/PIC.h>
@@ -70,6 +71,7 @@ namespace Kernel
 
 	void PIC::eoi(uint8_t irq)
 	{
+		ASSERT(!interrupts_enabled());
 		if (irq >= 8)
 			IO::outb(PIC2_CMD, PIC_EOI);
 		IO::outb(PIC1_CMD, PIC_EOI);
@@ -77,6 +79,10 @@ namespace Kernel
 
 	void PIC::enable_irq(uint8_t irq)
 	{
+		CriticalScope _;
+		ASSERT(irq < 16);
+		ASSERT(m_reserved_irqs & (1 << irq));
+
 		uint16_t port = PIC1_DATA;
 		if(irq >= 8)
 		{
@@ -86,6 +92,37 @@ namespace Kernel
 		IO::outb(port, IO::inb(port) & ~(1 << irq));
 	}
 
+	BAN::ErrorOr<void> PIC::reserve_irq(uint8_t irq)
+	{
+		if (irq >= 16)
+		{
+			dwarnln("PIC only supports 16 irqs");
+			return BAN::Error::from_errno(EFAULT);
+		}
+		CriticalScope _;
+		if (m_reserved_irqs & (1 << irq))
+		{
+			dwarnln("irq {} is already reserved", irq);
+			return BAN::Error::from_errno(EFAULT);
+		}
+		m_reserved_irqs |= 1 << irq;
+		return {};
+	}
+
+	BAN::Optional<uint8_t> PIC::get_free_irq()
+	{
+		CriticalScope _;
+		for (int irq = 0; irq < 16; irq++)
+		{
+			if (m_reserved_irqs & (1 << irq))
+				continue;
+			m_reserved_irqs |= 1 << irq;
+			return irq;
+		}
+		
+		return {};
+	}
+
 	bool PIC::is_in_service(uint8_t irq)
 	{
 		uint16_t port = PIC1_CMD;

kernel/kernel/Storage/ATA/AHCI/Controller.cpp

@@ -25,8 +25,8 @@ namespace Kernel
 
 		// Enable interrupts and bus mastering
 		m_pci_device.enable_bus_mastering();
-		m_pci_device.enable_pin_interrupts();
+		TRY(m_pci_device.reserve_irqs(1));
-		set_irq(TRY(m_pci_device.get_irq()));
+		set_irq(m_pci_device.get_irq(0));
 		enable_interrupt();
 		abar_mem.ghc = abar_mem.ghc | SATA_GHC_INTERRUPT_ENABLE;
 

kernel/kernel/Storage/ATA/ATABus.cpp

@@ -21,8 +21,6 @@ namespace Kernel
 		auto bus = BAN::RefPtr<ATABus>::adopt(bus_ptr);
 		bus->set_irq(irq);
 		TRY(bus->initialize());
-		if (bus->m_devices.empty())
-			return BAN::Error::from_errno(ENODEV);
 		return bus;
 	}
 

kernel/kernel/Storage/ATA/ATAController.cpp

@@ -41,6 +41,8 @@ namespace Kernel
 
 		uint8_t prog_if = m_pci_device.read_byte(0x09);
 
+		// FIXME: support native mode
+
 		if ((prog_if & ATA_PROGIF_CAN_MODIFY_PRIMARY_NATIVE) && (prog_if & ATA_PROGIF_PRIMARY_NATIVE))
 		{
 			prog_if &= ~ATA_PROGIF_PRIMARY_NATIVE;
@@ -57,11 +59,16 @@ namespace Kernel
 
 		if (!(prog_if & ATA_PROGIF_PRIMARY_NATIVE))
 		{
-			auto bus_or_error = ATABus::create(0x1F0, 0x3F6, 14);
+			if (InterruptController::get().reserve_irq(14).is_error())
-			if (bus_or_error.is_error())
+				dwarnln("Could not reserve interrupt {} for ATA device", 14);
-				dprintln("IDE ATABus: {}", bus_or_error.error());
 			else
-				TRY(buses.push_back(bus_or_error.release_value()));
+			{
+				auto bus_or_error = ATABus::create(0x1F0, 0x3F6, 14);
+				if (bus_or_error.is_error())
+					dprintln("IDE ATABus: {}", bus_or_error.error());
+				else
+					TRY(buses.push_back(bus_or_error.release_value()));
+			}
 		}
 		else
 		{
@@ -70,11 +77,16 @@ namespace Kernel
 
 		if (!(prog_if & ATA_PROGIF_SECONDARY_NATIVE))
 		{
-			auto bus_or_error = ATABus::create(0x170, 0x376, 15);
+			if (InterruptController::get().reserve_irq(15).is_error())
-			if (bus_or_error.is_error())
+				dwarnln("Could not reserver interrupt {} for ATA device", 15);
-				dprintln("IDE ATABus: {}", bus_or_error.error());
 			else
-				TRY(buses.push_back(bus_or_error.release_value()));
+			{
+				auto bus_or_error = ATABus::create(0x170, 0x376, 15);
+				if (bus_or_error.is_error())
+					dprintln("IDE ATABus: {}", bus_or_error.error());
+				else
+					TRY(buses.push_back(bus_or_error.release_value()));
+			}
 		}
 		else
 		{

kernel/kernel/Terminal/Serial.cpp

@@ -195,12 +195,14 @@ namespace Kernel
 		if (serial.port() == COM1_PORT)
 		{
 			IO::outb(COM1_PORT + 1, 1);
+			TRY(InterruptController::get().reserve_irq(COM1_IRQ));
 			tty->set_irq(COM1_IRQ);
 			tty->enable_interrupt();
 		}
 		else if (serial.port() == COM2_PORT)
 		{
 			IO::outb(COM2_PORT + 1, 1);
+			TRY(InterruptController::get().reserve_irq(COM2_IRQ));
 			tty->set_irq(COM2_IRQ);
 			tty->enable_interrupt();
 		}

kernel/kernel/Timer/HPET.cpp

@@ -105,6 +105,7 @@ namespace Kernel
 				if (irq_cap & (1 << irq))
 					break;
 		}
+		TRY(InterruptController::get().reserve_irq(irq));
 
 		unmapper.disable();
 

kernel/kernel/kernel.cpp

@@ -182,17 +182,19 @@ static void init2(void*)
 	SystemTimer::get().sleep(5000);
 #endif
 
+	// Initialize empty keymap
+	MUST(Input::KeyboardLayout::initialize());
+	if (auto res = PS2Controller::initialize(); res.is_error())
+		dprintln("{}", res.error());
+
+	// NOTE: PCI devices are the last ones to be initialized
+	//       so other devices can reserve predefined interrupts
 	PCI::PCIManager::initialize();
 	dprintln("PCI initialized");
 
 	VirtualFileSystem::initialize(cmdline.root);
 	dprintln("VFS initialized");
 
-	// Initialize empty keymap
-	MUST(Input::KeyboardLayout::initialize());
-	if (auto res = PS2Controller::initialize(); res.is_error())
-		dprintln("{}", res.error());
-
 	TTY::initialize_devices();
 
 	MUST(Process::create_userspace({ 0, 0, 0, 0 }, "/usr/bin/init"sv));