From c6130f33d789c611c3374eed9d5cea9590525a15 Mon Sep 17 00:00:00 2001 From: Bananymous Date: Sat, 13 Jan 2024 17:11:26 +0200 Subject: [PATCH] Kernel: Implement MSI, MSI-X and interrupt reservation --- kernel/include/kernel/APIC.h | 4 + kernel/include/kernel/InterruptController.h | 6 + kernel/include/kernel/PCI.h | 12 +- kernel/include/kernel/PIC.h | 4 + kernel/kernel/APIC.cpp | 71 ++++++++ kernel/kernel/Input/PS2/Controller.cpp | 12 ++ kernel/kernel/PCI.cpp | 155 +++++++++++++++--- kernel/kernel/PIC.cpp | 37 +++++ kernel/kernel/Storage/ATA/AHCI/Controller.cpp | 4 +- kernel/kernel/Storage/ATA/ATABus.cpp | 2 - kernel/kernel/Storage/ATA/ATAController.cpp | 28 +++- kernel/kernel/Terminal/Serial.cpp | 2 + kernel/kernel/Timer/HPET.cpp | 1 + kernel/kernel/kernel.cpp | 12 +- 14 files changed, 310 insertions(+), 40 deletions(-) diff --git a/kernel/include/kernel/APIC.h b/kernel/include/kernel/APIC.h index 02db2597..d1a5f836 100644 --- a/kernel/include/kernel/APIC.h +++ b/kernel/include/kernel/APIC.h @@ -14,6 +14,9 @@ namespace Kernel virtual void enable_irq(uint8_t) override; virtual bool is_in_service(uint8_t) override; + virtual BAN::ErrorOr reserve_irq(uint8_t irq) override; + virtual BAN::Optional get_free_irq() override; + private: uint32_t read_from_local_apic(ptrdiff_t); void write_to_local_apic(ptrdiff_t, uint32_t); @@ -54,6 +57,7 @@ namespace Kernel Kernel::vaddr_t m_local_apic_vaddr = 0; BAN::Vector m_io_apics; uint8_t m_irq_overrides[0x100] {}; + uint8_t m_reserved_gsis[0x100 / 8] {}; }; } diff --git a/kernel/include/kernel/InterruptController.h b/kernel/include/kernel/InterruptController.h index 505f0343..8896d82a 100644 --- a/kernel/include/kernel/InterruptController.h +++ b/kernel/include/kernel/InterruptController.h @@ -1,5 +1,8 @@ #pragma once +#include +#include + #include #define DISABLE_INTERRUPTS() asm volatile("cli") @@ -37,6 +40,9 @@ namespace Kernel static void initialize(bool force_pic); static InterruptController& get(); + virtual BAN::ErrorOr reserve_irq(uint8_t irq) = 0; + virtual BAN::Optional get_free_irq() = 0; + bool is_using_apic() const { return m_using_apic; } void enter_acpi_mode(); diff --git a/kernel/include/kernel/PCI.h b/kernel/include/kernel/PCI.h index 64fd7167..4c47aa96 100644 --- a/kernel/include/kernel/PCI.h +++ b/kernel/include/kernel/PCI.h @@ -79,7 +79,8 @@ namespace Kernel::PCI uint16_t vendor_id() const { return m_vendor_id; } uint16_t device_id() const { return m_device_id; } - BAN::ErrorOr get_irq(); + BAN::ErrorOr reserve_irqs(uint8_t count); + uint8_t get_irq(uint8_t index); BAN::ErrorOr> allocate_bar_region(uint8_t bar_num); @@ -92,15 +93,15 @@ namespace Kernel::PCI void enable_io_space(); void disable_io_space(); - void enable_pin_interrupts(); - void disable_pin_interrupts(); - private: void enumerate_capabilites(); void set_command_bits(uint16_t mask); void unset_command_bits(uint16_t mask); + void enable_pin_interrupts(); + void disable_pin_interrupts(); + private: const uint8_t m_bus; const uint8_t m_dev; @@ -114,6 +115,9 @@ namespace Kernel::PCI uint16_t m_vendor_id; uint16_t m_device_id; + uint32_t m_reserved_irqs { 0 }; + uint8_t m_reserved_irq_count { 0 }; + BAN::Optional m_offset_msi; BAN::Optional m_offset_msi_x; }; diff --git a/kernel/include/kernel/PIC.h b/kernel/include/kernel/PIC.h index 02b016c9..b5da20a6 100644 --- a/kernel/include/kernel/PIC.h +++ b/kernel/include/kernel/PIC.h @@ -12,12 +12,16 @@ namespace Kernel virtual void enable_irq(uint8_t) override; virtual bool is_in_service(uint8_t) override; + virtual BAN::ErrorOr reserve_irq(uint8_t irq) override; + virtual BAN::Optional get_free_irq() override; + static void remap(); static void mask_all(); private: static PIC* create(); friend class InterruptController; + uint16_t m_reserved_irqs { 0 }; }; } diff --git a/kernel/kernel/APIC.cpp b/kernel/kernel/APIC.cpp index 1af40d70..568b0b5f 100644 --- a/kernel/kernel/APIC.cpp +++ b/kernel/kernel/APIC.cpp @@ -223,8 +223,16 @@ namespace Kernel void APIC::enable_irq(uint8_t irq) { + CriticalScope _; + uint32_t gsi = m_irq_overrides[irq]; + { + int byte = gsi / 8; + int bit = gsi % 8; + ASSERT(m_reserved_gsis[byte] & (1 << bit)); + } + IOAPIC* ioapic = nullptr; for (IOAPIC& io : m_io_apics) { @@ -258,4 +266,67 @@ namespace Kernel return isr & (1 << bit); } + BAN::ErrorOr APIC::reserve_irq(uint8_t irq) + { + CriticalScope _; + + uint32_t gsi = m_irq_overrides[irq]; + + IOAPIC* ioapic = nullptr; + for (IOAPIC& io : m_io_apics) + { + if (io.gsi_base <= gsi && gsi <= io.gsi_base + io.max_redirs) + { + ioapic = &io; + break; + } + } + + if (!ioapic) + { + dwarnln("Cannot enable irq {} for APIC", irq); + return BAN::Error::from_errno(EFAULT); + } + + int byte = gsi / 8; + int bit = gsi % 8; + if (m_reserved_gsis[byte] & (1 << bit)) + { + dwarnln("irq {} is already reserved", irq); + return BAN::Error::from_errno(EFAULT); + } + m_reserved_gsis[byte] |= 1 << bit; + return {}; + } + + BAN::Optional APIC::get_free_irq() + { + CriticalScope _; + for (int irq = 0; irq <= 0xFF; irq++) + { + uint32_t gsi = m_irq_overrides[irq]; + + IOAPIC* ioapic = nullptr; + for (IOAPIC& io : m_io_apics) + { + if (io.gsi_base <= gsi && gsi <= io.gsi_base + io.max_redirs) + { + ioapic = &io; + break; + } + } + + if (!ioapic) + continue; + + int byte = gsi / 8; + int bit = gsi % 8; + if (m_reserved_gsis[byte] & (1 << bit)) + continue; + m_reserved_gsis[byte] |= 1 << bit; + return irq; + } + return {}; + } + } diff --git a/kernel/kernel/Input/PS2/Controller.cpp b/kernel/kernel/Input/PS2/Controller.cpp index 9ae366d6..86a19c4a 100644 --- a/kernel/kernel/Input/PS2/Controller.cpp +++ b/kernel/kernel/Input/PS2/Controller.cpp @@ -329,6 +329,18 @@ namespace Kernel::Input } } + // Reserve IRQs + if (m_devices[0] && InterruptController::get().reserve_irq(PS2::IRQ::DEVICE0).is_error()) + { + dwarnln("Could not reserve irq for PS/2 port 1"); + m_devices[0].clear(); + } + if (m_devices[1] && InterruptController::get().reserve_irq(PS2::IRQ::DEVICE1).is_error()) + { + dwarnln("Could not reserve irq for PS/2 port 2"); + m_devices[1].clear(); + } + if (!m_devices[0] && !m_devices[1]) return {}; diff --git a/kernel/kernel/PCI.cpp b/kernel/kernel/PCI.cpp index c7d75b05..5b50d110 100644 --- a/kernel/kernel/PCI.cpp +++ b/kernel/kernel/PCI.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -6,8 +7,6 @@ #include #include -#include - #define INVALID_VENDOR 0xFFFF #define MULTI_FUNCTION 0x80 @@ -25,13 +24,22 @@ #define PCI_CMD_BUS_MASTER (1 << 2) #define PCI_CMD_INTERRUPT_DISABLE (1 << 10) -#define DEBUG_PCI 1 +#define DEBUG_PCI 0 namespace Kernel::PCI { static PCIManager* s_instance = nullptr; + struct MSIXEntry + { + uint32_t msg_addr_low; + uint32_t msg_addr_high; + uint32_t msg_data; + uint32_t vector_ctrl; + }; + static_assert(sizeof(MSIXEntry) == 16); + uint32_t PCIManager::read_config_dword(uint8_t bus, uint8_t dev, uint8_t func, uint8_t offset) { ASSERT(offset % 4 == 0); @@ -421,38 +429,147 @@ namespace Kernel::PCI } } - BAN::ErrorOr PCI::Device::get_irq() + BAN::ErrorOr PCI::Device::reserve_irqs(uint8_t count) { + if (m_offset_msi_x.has_value()) + { + uint16_t msg_ctrl = read_word(*m_offset_msi_x + 0x02); + if (count > (msg_ctrl & 0x7FF) + 1) + { + dwarnln("MSI-X: could not allocate {} interrupts, only {} supported", count, (msg_ctrl & 0x7FF) + 1); + return BAN::Error::from_errno(EFAULT); + } + msg_ctrl |= 1 << 15; // Enable + write_word(*m_offset_msi_x + 0x02, msg_ctrl); + disable_pin_interrupts(); + } + else if (m_offset_msi.has_value()) + { + if (count > 1) + { + dwarnln("MSI: could not allocate {} interrupts, (currently) only {} supported", count, 1); + return BAN::Error::from_errno(EFAULT); + } + uint16_t msg_ctrl = read_word(*m_offset_msi + 0x02); + msg_ctrl &= ~(0x07 << 4); // Only one interrupt + msg_ctrl |= 1u << 0; // Enable + write_word(*m_offset_msi + 0x02, msg_ctrl); + disable_pin_interrupts(); + } + else if (!InterruptController::get().is_using_apic()) + { + if (count > 1) + { + dwarnln("PIC: could not allocate {} interrupts, (currently) only {} supported", count, 1); + return BAN::Error::from_errno(EFAULT); + } + enable_pin_interrupts(); + } + else + { + ASSERT_NOT_REACHED(); + } + + for (; m_reserved_irq_count < count; m_reserved_irq_count++) + { + auto irq = InterruptController::get().get_free_irq(); + if (!irq.has_value()) + { + dwarnln("Could not reserve interrupt for PCI {}:{}.{}", m_bus, m_dev, m_func); + return BAN::Error::from_errno(EFAULT); + } + + ASSERT(*irq < 32); + ASSERT(!(m_reserved_irqs & (1 << *irq))); + m_reserved_irqs |= 1 << *irq; + } + + return {}; + } + + static uint64_t msi_message_address() + { + return 0xFEE00000; + } + + static uint32_t msi_message_data(uint8_t irq) + { + return (IRQ_VECTOR_BASE + irq) & 0xFF; + } + + uint8_t PCI::Device::get_irq(uint8_t index) + { + ASSERT(m_offset_msi.has_value() || m_offset_msi_x.has_value() || !InterruptController::get().is_using_apic()); + ASSERT(index < m_reserved_irq_count); + + uint8_t count_found = 0; + uint8_t irq = 0xFF; + for (uint8_t i = 0; i < 32; i++) + { + if (m_reserved_irqs & (1 << i)) + count_found++; + if (count_found > index) + { + irq = i; + break; + } + } + ASSERT(irq != 0xFF); + // Legacy PIC just uses the interrupt line field if (!InterruptController::get().is_using_apic()) - return read_byte(PCI_REG_IRQ_LINE); - - // TODO: use MSI and MSI-X if supported - - if (m_offset_msi.has_value()) { + write_byte(PCI_REG_IRQ_LINE, irq); + return irq; } if (m_offset_msi_x.has_value()) { + uint32_t dword0 = read_dword(*m_offset_msi_x); + ASSERT((dword0 & 0xFF) == 0x11); + + uint32_t dword1 = read_dword(*m_offset_msi_x + 0x04); + uint32_t offset = dword1 & ~3u; + uint8_t bir = dword1 & 3u; + + uint64_t msg_addr = msi_message_address(); + uint32_t msg_data = msi_message_data(irq); + + auto bar = MUST(allocate_bar_region(bir)); + ASSERT(bar->type() == BarType::MEM); + auto& msi_x_entry = reinterpret_cast(bar->vaddr() + offset)[index]; + msi_x_entry.msg_addr_low = msg_addr & 0xFFFFFFFF; + msi_x_entry.msg_addr_high = msg_addr >> 32;; + msi_x_entry.msg_data = msg_data; + msi_x_entry.vector_ctrl = msi_x_entry.vector_ctrl & ~1u; + + return irq; } - for (uint8_t irq_pin = 1; irq_pin <= 4; irq_pin++) + if (m_offset_msi.has_value()) { - acpi_resource_t dest; - auto err = lai_pci_route_pin(&dest, 0, m_bus, m_dev, m_func, irq_pin); - if (err != LAI_ERROR_NONE) + uint32_t dword0 = read_dword(*m_offset_msi); + ASSERT((dword0 & 0xFF) == 0x05); + + uint64_t msg_addr = msi_message_address(); + uint32_t msg_data = msi_message_data(irq); + + if (dword0 & (1 << 23)) { - dprintln("{}", lai_api_error_to_string(err)); - continue; + write_dword(*m_offset_msi + 0x04, msg_addr & 0xFFFFFFFF); + write_dword(*m_offset_msi + 0x08, msg_addr >> 32); + write_word(*m_offset_msi + 0x12, msg_data); + } + else + { + write_dword(*m_offset_msi + 0x04, msg_addr & 0xFFFFFFFF); + write_word(*m_offset_msi + 0x08, msg_data); } - write_byte(PCI_REG_IRQ_PIN, irq_pin); - return dest.base; + return irq; } - dwarnln("Could not allocate interrupt for PCI {}:{}.{}", m_bus, m_dev, m_func); - return BAN::Error::from_errno(ENOTSUP); + ASSERT_NOT_REACHED(); } void PCI::Device::set_command_bits(uint16_t mask) diff --git a/kernel/kernel/PIC.cpp b/kernel/kernel/PIC.cpp index 1ac8bf7f..8e079b72 100644 --- a/kernel/kernel/PIC.cpp +++ b/kernel/kernel/PIC.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -70,6 +71,7 @@ namespace Kernel void PIC::eoi(uint8_t irq) { + ASSERT(!interrupts_enabled()); if (irq >= 8) IO::outb(PIC2_CMD, PIC_EOI); IO::outb(PIC1_CMD, PIC_EOI); @@ -77,6 +79,10 @@ namespace Kernel void PIC::enable_irq(uint8_t irq) { + CriticalScope _; + ASSERT(irq < 16); + ASSERT(m_reserved_irqs & (1 << irq)); + uint16_t port = PIC1_DATA; if(irq >= 8) { @@ -86,6 +92,37 @@ namespace Kernel IO::outb(port, IO::inb(port) & ~(1 << irq)); } + BAN::ErrorOr PIC::reserve_irq(uint8_t irq) + { + if (irq >= 16) + { + dwarnln("PIC only supports 16 irqs"); + return BAN::Error::from_errno(EFAULT); + } + CriticalScope _; + if (m_reserved_irqs & (1 << irq)) + { + dwarnln("irq {} is already reserved", irq); + return BAN::Error::from_errno(EFAULT); + } + m_reserved_irqs |= 1 << irq; + return {}; + } + + BAN::Optional PIC::get_free_irq() + { + CriticalScope _; + for (int irq = 0; irq < 16; irq++) + { + if (m_reserved_irqs & (1 << irq)) + continue; + m_reserved_irqs |= 1 << irq; + return irq; + } + + return {}; + } + bool PIC::is_in_service(uint8_t irq) { uint16_t port = PIC1_CMD; diff --git a/kernel/kernel/Storage/ATA/AHCI/Controller.cpp b/kernel/kernel/Storage/ATA/AHCI/Controller.cpp index 23f81e00..9280ebca 100644 --- a/kernel/kernel/Storage/ATA/AHCI/Controller.cpp +++ b/kernel/kernel/Storage/ATA/AHCI/Controller.cpp @@ -25,8 +25,8 @@ namespace Kernel // Enable interrupts and bus mastering m_pci_device.enable_bus_mastering(); - m_pci_device.enable_pin_interrupts(); - set_irq(TRY(m_pci_device.get_irq())); + TRY(m_pci_device.reserve_irqs(1)); + set_irq(m_pci_device.get_irq(0)); enable_interrupt(); abar_mem.ghc = abar_mem.ghc | SATA_GHC_INTERRUPT_ENABLE; diff --git a/kernel/kernel/Storage/ATA/ATABus.cpp b/kernel/kernel/Storage/ATA/ATABus.cpp index c004fd4f..d006ad70 100644 --- a/kernel/kernel/Storage/ATA/ATABus.cpp +++ b/kernel/kernel/Storage/ATA/ATABus.cpp @@ -21,8 +21,6 @@ namespace Kernel auto bus = BAN::RefPtr::adopt(bus_ptr); bus->set_irq(irq); TRY(bus->initialize()); - if (bus->m_devices.empty()) - return BAN::Error::from_errno(ENODEV); return bus; } diff --git a/kernel/kernel/Storage/ATA/ATAController.cpp b/kernel/kernel/Storage/ATA/ATAController.cpp index 1abc317a..0b18c0fb 100644 --- a/kernel/kernel/Storage/ATA/ATAController.cpp +++ b/kernel/kernel/Storage/ATA/ATAController.cpp @@ -41,6 +41,8 @@ namespace Kernel uint8_t prog_if = m_pci_device.read_byte(0x09); + // FIXME: support native mode + if ((prog_if & ATA_PROGIF_CAN_MODIFY_PRIMARY_NATIVE) && (prog_if & ATA_PROGIF_PRIMARY_NATIVE)) { prog_if &= ~ATA_PROGIF_PRIMARY_NATIVE; @@ -57,11 +59,16 @@ namespace Kernel if (!(prog_if & ATA_PROGIF_PRIMARY_NATIVE)) { - auto bus_or_error = ATABus::create(0x1F0, 0x3F6, 14); - if (bus_or_error.is_error()) - dprintln("IDE ATABus: {}", bus_or_error.error()); + if (InterruptController::get().reserve_irq(14).is_error()) + dwarnln("Could not reserve interrupt {} for ATA device", 14); else - TRY(buses.push_back(bus_or_error.release_value())); + { + auto bus_or_error = ATABus::create(0x1F0, 0x3F6, 14); + if (bus_or_error.is_error()) + dprintln("IDE ATABus: {}", bus_or_error.error()); + else + TRY(buses.push_back(bus_or_error.release_value())); + } } else { @@ -70,11 +77,16 @@ namespace Kernel if (!(prog_if & ATA_PROGIF_SECONDARY_NATIVE)) { - auto bus_or_error = ATABus::create(0x170, 0x376, 15); - if (bus_or_error.is_error()) - dprintln("IDE ATABus: {}", bus_or_error.error()); + if (InterruptController::get().reserve_irq(15).is_error()) + dwarnln("Could not reserver interrupt {} for ATA device", 15); else - TRY(buses.push_back(bus_or_error.release_value())); + { + auto bus_or_error = ATABus::create(0x170, 0x376, 15); + if (bus_or_error.is_error()) + dprintln("IDE ATABus: {}", bus_or_error.error()); + else + TRY(buses.push_back(bus_or_error.release_value())); + } } else { diff --git a/kernel/kernel/Terminal/Serial.cpp b/kernel/kernel/Terminal/Serial.cpp index f756fcbd..b4b97e18 100644 --- a/kernel/kernel/Terminal/Serial.cpp +++ b/kernel/kernel/Terminal/Serial.cpp @@ -195,12 +195,14 @@ namespace Kernel if (serial.port() == COM1_PORT) { IO::outb(COM1_PORT + 1, 1); + TRY(InterruptController::get().reserve_irq(COM1_IRQ)); tty->set_irq(COM1_IRQ); tty->enable_interrupt(); } else if (serial.port() == COM2_PORT) { IO::outb(COM2_PORT + 1, 1); + TRY(InterruptController::get().reserve_irq(COM2_IRQ)); tty->set_irq(COM2_IRQ); tty->enable_interrupt(); } diff --git a/kernel/kernel/Timer/HPET.cpp b/kernel/kernel/Timer/HPET.cpp index 59089e90..454ffddb 100644 --- a/kernel/kernel/Timer/HPET.cpp +++ b/kernel/kernel/Timer/HPET.cpp @@ -105,6 +105,7 @@ namespace Kernel if (irq_cap & (1 << irq)) break; } + TRY(InterruptController::get().reserve_irq(irq)); unmapper.disable(); diff --git a/kernel/kernel/kernel.cpp b/kernel/kernel/kernel.cpp index 835bfd1a..c4db1e19 100644 --- a/kernel/kernel/kernel.cpp +++ b/kernel/kernel/kernel.cpp @@ -182,17 +182,19 @@ static void init2(void*) SystemTimer::get().sleep(5000); #endif + // Initialize empty keymap + MUST(Input::KeyboardLayout::initialize()); + if (auto res = PS2Controller::initialize(); res.is_error()) + dprintln("{}", res.error()); + + // NOTE: PCI devices are the last ones to be initialized + // so other devices can reserve predefined interrupts PCI::PCIManager::initialize(); dprintln("PCI initialized"); VirtualFileSystem::initialize(cmdline.root); dprintln("VFS initialized"); - // Initialize empty keymap - MUST(Input::KeyboardLayout::initialize()); - if (auto res = PS2Controller::initialize(); res.is_error()) - dprintln("{}", res.error()); - TTY::initialize_devices(); MUST(Process::create_userspace({ 0, 0, 0, 0 }, "/usr/bin/init"sv));