diff --git a/kernel/arch/i686/interrupts.S b/kernel/arch/i686/interrupts.S index 6838ef537f..e0d70da6aa 100644 --- a/kernel/arch/i686/interrupts.S +++ b/kernel/arch/i686/interrupts.S @@ -218,3 +218,179 @@ irq 28 irq 29 irq 30 irq 31 +irq 32 +irq 33 +irq 34 +irq 35 +irq 36 +irq 37 +irq 38 +irq 39 +irq 40 +irq 41 +irq 42 +irq 43 +irq 44 +irq 45 +irq 46 +irq 47 +irq 48 +irq 49 +irq 50 +irq 51 +irq 52 +irq 53 +irq 54 +irq 55 +irq 56 +irq 57 +irq 58 +irq 59 +irq 60 +irq 61 +irq 62 +irq 63 +irq 64 +irq 65 +irq 66 +irq 67 +irq 68 +irq 69 +irq 70 +irq 71 +irq 72 +irq 73 +irq 74 +irq 75 +irq 76 +irq 77 +irq 78 +irq 79 +irq 80 +irq 81 +irq 82 +irq 83 +irq 84 +irq 85 +irq 86 +irq 87 +irq 88 +irq 89 +irq 90 +irq 91 +irq 92 +irq 93 +irq 94 +irq 95 +irq 96 +irq 97 +irq 98 +irq 99 +irq 100 +irq 101 +irq 102 +irq 103 +irq 104 +irq 105 +irq 106 +irq 107 +irq 108 +irq 109 +irq 110 +irq 111 +irq 112 +irq 113 +irq 114 +irq 115 +irq 116 +irq 117 +irq 118 +irq 119 +irq 120 +irq 121 +irq 122 +irq 123 +irq 124 +irq 125 +irq 126 +irq 127 +irq 128 +irq 129 +irq 130 +irq 131 +irq 132 +irq 133 +irq 134 +irq 135 +irq 136 +irq 137 +irq 138 +irq 139 +irq 140 +irq 141 +irq 142 +irq 143 +irq 144 +irq 145 +irq 146 +irq 147 +irq 148 +irq 149 +irq 150 +irq 151 +irq 152 +irq 153 +irq 154 +irq 155 +irq 156 +irq 157 +irq 158 +irq 159 +irq 160 +irq 161 +irq 162 +irq 163 +irq 164 +irq 165 +irq 166 +irq 167 +irq 168 +irq 169 +irq 170 +irq 171 +irq 172 +irq 173 +irq 174 +irq 175 +irq 176 +irq 177 +irq 178 +irq 179 +irq 180 +irq 181 +irq 182 +irq 183 +irq 184 +irq 185 +irq 186 +irq 187 +irq 188 +irq 189 +irq 190 +irq 191 +irq 192 +irq 193 +irq 194 +irq 195 +irq 196 +irq 197 +irq 198 +irq 199 +irq 200 +irq 201 +irq 202 +irq 203 +irq 204 +irq 205 +irq 206 +irq 207 diff --git a/kernel/arch/x86_64/interrupts.S b/kernel/arch/x86_64/interrupts.S index 35862f758a..b20c143beb 100644 --- a/kernel/arch/x86_64/interrupts.S +++ b/kernel/arch/x86_64/interrupts.S @@ -172,3 +172,179 @@ irq 28 irq 29 irq 30 irq 31 +irq 32 +irq 33 +irq 34 +irq 35 +irq 36 +irq 37 +irq 38 +irq 39 +irq 40 +irq 41 +irq 42 +irq 43 +irq 44 +irq 45 +irq 46 +irq 47 +irq 48 +irq 49 +irq 50 +irq 51 +irq 52 +irq 53 +irq 54 +irq 55 +irq 56 +irq 57 +irq 58 +irq 59 +irq 60 +irq 61 +irq 62 +irq 63 +irq 64 +irq 65 +irq 66 +irq 67 +irq 68 +irq 69 +irq 70 +irq 71 +irq 72 +irq 73 +irq 74 +irq 75 +irq 76 +irq 77 +irq 78 +irq 79 +irq 80 +irq 81 +irq 82 +irq 83 +irq 84 +irq 85 +irq 86 +irq 87 +irq 88 +irq 89 +irq 90 +irq 91 +irq 92 +irq 93 +irq 94 +irq 95 +irq 96 +irq 97 +irq 98 +irq 99 +irq 100 +irq 101 +irq 102 +irq 103 +irq 104 +irq 105 +irq 106 +irq 107 +irq 108 +irq 109 +irq 110 +irq 111 +irq 112 +irq 113 +irq 114 +irq 115 +irq 116 +irq 117 +irq 118 +irq 119 +irq 120 +irq 121 +irq 122 +irq 123 +irq 124 +irq 125 +irq 126 +irq 127 +irq 128 +irq 129 +irq 130 +irq 131 +irq 132 +irq 133 +irq 134 +irq 135 +irq 136 +irq 137 +irq 138 +irq 139 +irq 140 +irq 141 +irq 142 +irq 143 +irq 144 +irq 145 +irq 146 +irq 147 +irq 148 +irq 149 +irq 150 +irq 151 +irq 152 +irq 153 +irq 154 +irq 155 +irq 156 +irq 157 +irq 158 +irq 159 +irq 160 +irq 161 +irq 162 +irq 163 +irq 164 +irq 165 +irq 166 +irq 167 +irq 168 +irq 169 +irq 170 +irq 171 +irq 172 +irq 173 +irq 174 +irq 175 +irq 176 +irq 177 +irq 178 +irq 179 +irq 180 +irq 181 +irq 182 +irq 183 +irq 184 +irq 185 +irq 186 +irq 187 +irq 188 +irq 189 +irq 190 +irq 191 +irq 192 +irq 193 +irq 194 +irq 195 +irq 196 +irq 197 +irq 198 +irq 199 +irq 200 +irq 201 +irq 202 +irq 203 +irq 204 +irq 205 +irq 206 +irq 207 diff --git a/kernel/include/kernel/APIC.h b/kernel/include/kernel/APIC.h index 5a7e9e1bbf..2054be6bfe 100644 --- a/kernel/include/kernel/APIC.h +++ b/kernel/include/kernel/APIC.h @@ -60,13 +60,15 @@ namespace Kernel }; private: + static constexpr uint8_t m_irq_count = IRQ_MSI_BASE - IRQ_VECTOR_BASE; + SpinLock m_lock; BAN::Vector m_processors; Kernel::paddr_t m_local_apic_paddr = 0; Kernel::vaddr_t m_local_apic_vaddr = 0; BAN::Vector m_io_apics; uint8_t m_irq_overrides[0x100] {}; - uint8_t m_reserved_gsis[0x100 / 8] {}; + uint8_t m_reserved_gsis[m_irq_count / 8] {}; uint64_t m_lapic_timer_frequency_hz { 0 }; }; diff --git a/kernel/include/kernel/IDT.h b/kernel/include/kernel/IDT.h index a3cf7506cc..cb7d200cb0 100644 --- a/kernel/include/kernel/IDT.h +++ b/kernel/include/kernel/IDT.h @@ -7,14 +7,22 @@ #include -constexpr uint8_t IRQ_VECTOR_BASE = 0x20; -constexpr uint8_t IRQ_YIELD = 32; -constexpr uint8_t IRQ_IPI = 33; -constexpr uint8_t IRQ_TIMER = 34; - namespace Kernel { + // IDT entries + // 0x00->0x1F (32): ISR + // 0x20->0x7F (96): PIC/IOAPIC + // 0x80->0xEF (112): MSI + // 0xF0->0xFE (15): internal + + constexpr uint8_t IRQ_VECTOR_BASE = 0x20; + constexpr uint8_t IRQ_MSI_BASE = 0x80; + constexpr uint8_t IRQ_SYSCALL = 0xF0; + constexpr uint8_t IRQ_YIELD = 0xF1; + constexpr uint8_t IRQ_IPI = 0xF2; + constexpr uint8_t IRQ_TIMER = 0xF3; + #if ARCH(x86_64) struct GateDescriptor { diff --git a/kernel/include/kernel/Interruptable.h b/kernel/include/kernel/Interruptable.h index 85f62471fb..cc37163c43 100644 --- a/kernel/include/kernel/Interruptable.h +++ b/kernel/include/kernel/Interruptable.h @@ -7,8 +7,6 @@ namespace Kernel { public: void set_irq(int irq); - void enable_interrupt(); - void disable_interrupt(); virtual void handle_irq() = 0; diff --git a/kernel/include/kernel/PCI.h b/kernel/include/kernel/PCI.h index 4fd8830887..dd6732e7e6 100644 --- a/kernel/include/kernel/PCI.h +++ b/kernel/include/kernel/PCI.h @@ -55,6 +55,15 @@ namespace Kernel::PCI class Device { + public: + enum class InterruptMechanism + { + NONE, + MSIX, + MSI, + PIN, + }; + public: Device() = default; @@ -84,8 +93,9 @@ namespace Kernel::PCI uint16_t vendor_id() const { return m_vendor_id; } uint16_t device_id() const { return m_device_id; } - BAN::ErrorOr reserve_irqs(uint8_t count); - uint8_t get_irq(uint8_t index); + uint8_t get_interrupt(uint8_t index) const; + BAN::ErrorOr reserve_interrupts(uint8_t count); + void enable_interrupt(uint8_t index, Interruptable&); BAN::ErrorOr> allocate_bar_region(uint8_t bar_num); @@ -123,8 +133,9 @@ namespace Kernel::PCI uint16_t m_vendor_id { 0 }; uint16_t m_device_id { 0 }; - uint32_t m_reserved_irqs { 0 }; - uint8_t m_reserved_irq_count { 0 }; + InterruptMechanism m_interrupt_mechanism { InterruptMechanism::NONE }; + uint8_t m_reserved_interrupts[0x100 / 8] {}; + uint8_t m_reserved_interrupt_count { 0 }; BAN::Optional m_offset_msi; BAN::Optional m_offset_msi_x; @@ -159,6 +170,8 @@ namespace Kernel::PCI void write_config_word(uint8_t bus, uint8_t dev, uint8_t func, uint8_t offset, uint16_t value); void write_config_byte(uint8_t bus, uint8_t dev, uint8_t func, uint8_t offset, uint8_t value); + BAN::Optional reserve_msi(); + private: PCIManager() : m_bus_pcie_paddr(0) {} void check_function(uint8_t bus, uint8_t dev, uint8_t func); @@ -168,10 +181,14 @@ namespace Kernel::PCI void initialize_impl(); private: + static constexpr uint8_t m_msi_count = IRQ_SYSCALL - IRQ_MSI_BASE; using PCIBus = BAN::Array, 32>; - BAN::Array m_buses; - BAN::Array m_bus_pcie_paddr; - bool m_is_pcie { false }; + BAN::Array m_buses; + BAN::Array m_bus_pcie_paddr; + bool m_is_pcie { false }; + + SpinLock m_reserved_msi_lock; + BAN::Array m_reserved_msi_bitmap; }; } diff --git a/kernel/include/kernel/Storage/NVMe/Queue.h b/kernel/include/kernel/Storage/NVMe/Queue.h index 1206c28c25..a4496ed064 100644 --- a/kernel/include/kernel/Storage/NVMe/Queue.h +++ b/kernel/include/kernel/Storage/NVMe/Queue.h @@ -13,7 +13,7 @@ namespace Kernel class NVMeQueue : public Interruptable { public: - NVMeQueue(BAN::UniqPtr&& cq, BAN::UniqPtr&& sq, volatile NVMe::DoorbellRegisters& db, uint32_t qdepth, uint8_t irq); + NVMeQueue(BAN::UniqPtr&& cq, BAN::UniqPtr&& sq, volatile NVMe::DoorbellRegisters& db, uint32_t qdepth); uint16_t submit_command(NVMe::SubmissionQueueEntry& sqe); @@ -31,11 +31,11 @@ namespace Kernel uint32_t m_cq_head { 0 }; uint16_t m_cq_valid_phase { 1 }; - ThreadBlocker m_thread_blocker; - SpinLock m_lock; - BAN::Atomic m_used_mask { 0 }; - BAN::Atomic m_done_mask { 0 }; - volatile uint16_t m_status_codes[64] { }; + ThreadBlocker m_thread_blocker; + SpinLock m_lock; + BAN::Atomic m_used_mask { 0 }; + BAN::Atomic m_done_mask { 0 }; + volatile uint16_t m_status_codes[64] { }; static constexpr size_t m_mask_bits = sizeof(size_t) * 8; }; diff --git a/kernel/include/kernel/Syscall.h b/kernel/include/kernel/Syscall.h index 2f53a5475c..73ccc4c962 100644 --- a/kernel/include/kernel/Syscall.h +++ b/kernel/include/kernel/Syscall.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -10,7 +11,16 @@ namespace Kernel ALWAYS_INLINE long syscall(int syscall, uintptr_t arg1 = 0, uintptr_t arg2 = 0, uintptr_t arg3 = 0, uintptr_t arg4 = 0, uintptr_t arg5 = 0) { long ret; - asm volatile("int $0x80" : "=a"(ret) : "a"(syscall), "b"((uintptr_t)arg1), "c"((uintptr_t)arg2), "d"((uintptr_t)arg3), "S"((uintptr_t)arg4), "D"((uintptr_t)arg5) : "memory"); + asm volatile("int %[irq]" + : "=a"(ret) + : [irq]"i"(IRQ_SYSCALL) + , "a"(syscall) + , "b"((uintptr_t)arg1) + , "c"((uintptr_t)arg2) + , "d"((uintptr_t)arg3) + , "S"((uintptr_t)arg4) + , "D"((uintptr_t)arg5) + : "memory"); return ret; } diff --git a/kernel/kernel/ACPI/ACPI.cpp b/kernel/kernel/ACPI/ACPI.cpp index f7959c927e..78053c9ac7 100644 --- a/kernel/kernel/ACPI/ACPI.cpp +++ b/kernel/kernel/ACPI/ACPI.cpp @@ -703,10 +703,9 @@ acpi_release_global_lock: ); } - - set_irq(irq); - enable_interrupt(); + InterruptController::get().enable_irq(irq); + Process::create_kernel([](void*) { get().acpi_event_task(); }, nullptr); } diff --git a/kernel/kernel/APIC.cpp b/kernel/kernel/APIC.cpp index 0b11a621d5..a847fac928 100644 --- a/kernel/kernel/APIC.cpp +++ b/kernel/kernel/APIC.cpp @@ -371,7 +371,7 @@ namespace Kernel | ICR_LO_level_assert | ICR_LO_trigger_mode_level | ICR_LO_destination_shorthand_none - | (IRQ_VECTOR_BASE + IRQ_IPI) + | IRQ_IPI ); } @@ -388,7 +388,7 @@ namespace Kernel | ICR_LO_level_assert | ICR_LO_trigger_mode_level | ICR_LO_destination_shorthand_all_excluding_self - | (IRQ_VECTOR_BASE + IRQ_IPI) + | IRQ_IPI ); } @@ -423,7 +423,7 @@ namespace Kernel dprintln("CPU {}: lapic timer frequency: {} Hz", Kernel::Processor::current_id(), m_lapic_timer_frequency_hz); - write_to_local_apic(LAPIC_TIMER_LVT, TimerModePeriodic | (IRQ_VECTOR_BASE + IRQ_TIMER)); + write_to_local_apic(LAPIC_TIMER_LVT, TimerModePeriodic | IRQ_TIMER); write_to_local_apic(LAPIC_TIMER_INITIAL_REG, m_lapic_timer_frequency_hz / 2 / 100); } @@ -536,9 +536,9 @@ namespace Kernel BAN::Optional APIC::get_free_irq() { SpinLockGuard _(m_lock); - for (int irq = 0; irq <= 0xFF; irq++) + for (uint8_t irq = 0; irq < m_irq_count; irq++) { - uint32_t gsi = m_irq_overrides[irq]; + const uint8_t gsi = m_irq_overrides[irq]; IOAPIC* ioapic = nullptr; for (IOAPIC& io : m_io_apics) @@ -553,8 +553,8 @@ namespace Kernel if (!ioapic) continue; - int byte = gsi / 8; - int bit = gsi % 8; + const uint8_t byte = gsi / 8; + const uint8_t bit = gsi % 8; if (m_reserved_gsis[byte] & (1 << bit)) continue; m_reserved_gsis[byte] |= 1 << bit; diff --git a/kernel/kernel/IDT.cpp b/kernel/kernel/IDT.cpp index 98236ced4d..d43ce70577 100644 --- a/kernel/kernel/IDT.cpp +++ b/kernel/kernel/IDT.cpp @@ -10,7 +10,15 @@ #include #define ISR_LIST_X X(0) X(1) X(2) X(3) X(4) X(5) X(6) X(7) X(8) X(9) X(10) X(11) X(12) X(13) X(14) X(15) X(16) X(17) X(18) X(19) X(20) X(21) X(22) X(23) X(24) X(25) X(26) X(27) X(28) X(29) X(30) X(31) -#define IRQ_LIST_X X(0) X(1) X(2) X(3) X(4) X(5) X(6) X(7) X(8) X(9) X(10) X(11) X(12) X(13) X(14) X(15) X(16) X(17) X(18) X(19) X(20) X(21) X(22) X(23) X(24) X(25) X(26) X(27) X(28) X(29) X(30) X(31) +#define IRQ_LIST_X X( 0) X( 1) X( 2) X( 3) X( 4) X( 5) X( 6) X( 7) X( 8) X( 9) X( 10) X( 11) X( 12) X( 13) X( 14) X( 15) X( 16) X( 17) X( 18) X( 19) X( 20) X( 21) X( 22) X( 23) X( 24) X( 25) X( 26) X( 27) X( 28) X( 29) X( 30) X( 31) \ + X( 32) X( 33) X( 34) X( 35) X( 36) X( 37) X( 38) X( 39) X( 40) X( 41) X( 42) X( 43) X( 44) X( 45) X( 46) X( 47) X( 48) X( 49) X( 50) X( 51) X( 52) X( 53) X( 54) X( 55) X( 56) X( 57) X( 58) X( 59) X( 60) X( 61) X( 62) X( 63) \ + X( 64) X( 65) X( 66) X( 67) X( 68) X( 69) X( 70) X( 71) X( 72) X( 73) X( 74) X( 75) X( 76) X( 77) X( 78) X( 79) X( 80) X( 81) X( 82) X( 83) X( 84) X( 85) X( 86) X( 87) X( 88) X( 89) X( 90) X( 91) X( 92) X( 93) X( 94) X( 95) \ + X( 96) X( 97) X( 98) X( 99) X(100) X(101) X(102) X(103) X(104) X(105) X(106) X(107) X(108) X(109) X(110) X(111) X(112) X(113) X(114) X(115) X(116) X(117) X(118) X(119) X(120) X(121) X(122) X(123) X(124) X(125) X(126) X(127) \ + X(128) X(129) X(130) X(131) X(132) X(133) X(134) X(135) X(136) X(137) X(138) X(139) X(140) X(141) X(142) X(143) X(144) X(145) X(146) X(147) X(148) X(149) X(150) X(151) X(152) X(153) X(154) X(155) X(156) X(157) X(158) X(159) \ + X(160) X(161) X(162) X(163) X(164) X(165) X(166) X(167) X(168) X(169) X(170) X(171) X(172) X(173) X(174) X(175) X(176) X(177) X(178) X(179) X(180) X(181) X(182) X(183) X(184) X(185) X(186) X(187) X(188) X(189) X(190) X(191) \ + X(192) X(193) X(194) X(195) X(196) X(197) X(198) X(199) X(200) X(201) X(202) X(203) X(204) X(205) X(206) X(207) + +static_assert(Kernel::IRQ_SYSCALL == Kernel::IRQ_VECTOR_BASE + 208); namespace Kernel { @@ -321,15 +329,15 @@ done: extern "C" void cpp_yield_handler(InterruptStack* interrupt_stack, InterruptRegisters* interrupt_registers) { // yield is raised through kernel software interrupt - ASSERT(!InterruptController::get().is_in_service(IRQ_YIELD)); + ASSERT(!InterruptController::get().is_in_service(IRQ_YIELD - IRQ_VECTOR_BASE)); ASSERT(!GDT::is_user_segment(interrupt_stack->cs)); Processor::scheduler().reschedule(interrupt_stack, interrupt_registers); } extern "C" void cpp_ipi_handler() { - ASSERT(InterruptController::get().is_in_service(IRQ_IPI)); - InterruptController::get().eoi(IRQ_IPI); + ASSERT(InterruptController::get().is_in_service(IRQ_IPI - IRQ_VECTOR_BASE)); + InterruptController::get().eoi(IRQ_IPI - IRQ_VECTOR_BASE); Processor::handle_ipi(); } @@ -343,8 +351,8 @@ done: asm volatile("cli; 1: hlt; jmp 1b"); } - ASSERT(InterruptController::get().is_in_service(IRQ_TIMER)); - InterruptController::get().eoi(IRQ_TIMER); + ASSERT(InterruptController::get().is_in_service(IRQ_TIMER - IRQ_VECTOR_BASE)); + InterruptController::get().eoi(IRQ_TIMER - IRQ_VECTOR_BASE); if (Processor::current_is_bsb()) Process::update_alarm_queue(); @@ -452,11 +460,10 @@ done: IRQ_LIST_X #undef X - idt->register_interrupt_handler(IRQ_VECTOR_BASE + IRQ_YIELD, asm_yield_handler); - idt->register_interrupt_handler(IRQ_VECTOR_BASE + IRQ_IPI, asm_ipi_handler); - idt->register_interrupt_handler(IRQ_VECTOR_BASE + IRQ_TIMER, asm_timer_handler); - - idt->register_syscall_handler(0x80, asm_syscall_handler); + idt->register_interrupt_handler(IRQ_YIELD, asm_yield_handler); + idt->register_interrupt_handler(IRQ_IPI, asm_ipi_handler); + idt->register_interrupt_handler(IRQ_TIMER, asm_timer_handler); + idt->register_syscall_handler(IRQ_SYSCALL, asm_syscall_handler); return idt; } diff --git a/kernel/kernel/Input/PS2/Controller.cpp b/kernel/kernel/Input/PS2/Controller.cpp index a927436314..685fba50a9 100644 --- a/kernel/kernel/Input/PS2/Controller.cpp +++ b/kernel/kernel/Input/PS2/Controller.cpp @@ -342,13 +342,13 @@ namespace Kernel::Input if (m_devices[0]) { m_devices[0]->set_irq(PS2::IRQ::DEVICE0); - m_devices[0]->enable_interrupt(); + InterruptController::get().enable_irq(PS2::IRQ::DEVICE0); config |= PS2::Config::INTERRUPT_FIRST_PORT; } if (m_devices[1]) { m_devices[1]->set_irq(PS2::IRQ::DEVICE1); - m_devices[1]->enable_interrupt(); + InterruptController::get().enable_irq(PS2::IRQ::DEVICE1); config |= PS2::Config::INTERRUPT_SECOND_PORT; } diff --git a/kernel/kernel/Interruptable.cpp b/kernel/kernel/Interruptable.cpp index 502c4b33b6..ae57015172 100644 --- a/kernel/kernel/Interruptable.cpp +++ b/kernel/kernel/Interruptable.cpp @@ -14,15 +14,4 @@ namespace Kernel Processor::idt().register_irq_handler(irq, this); } - void Interruptable::enable_interrupt() - { - ASSERT(m_irq != -1); - InterruptController::get().enable_irq(m_irq); - } - - void Interruptable::disable_interrupt() - { - ASSERT_NOT_REACHED(); - } - } diff --git a/kernel/kernel/Networking/E1000/E1000.cpp b/kernel/kernel/Networking/E1000/E1000.cpp index 4077a33e34..03831867ca 100644 --- a/kernel/kernel/Networking/E1000/E1000.cpp +++ b/kernel/kernel/Networking/E1000/E1000.cpp @@ -245,9 +245,8 @@ namespace Kernel write32(REG_IMS, IMC_RxQ0); read32(REG_ICR); - TRY(m_pci_device.reserve_irqs(1)); - set_irq(m_pci_device.get_irq(0)); - Interruptable::enable_interrupt(); + TRY(m_pci_device.reserve_interrupts(1)); + m_pci_device.enable_interrupt(0, *this); return {}; } diff --git a/kernel/kernel/Networking/RTL8169/RTL8169.cpp b/kernel/kernel/Networking/RTL8169/RTL8169.cpp index 0c165481ba..5fd47e79ad 100644 --- a/kernel/kernel/Networking/RTL8169/RTL8169.cpp +++ b/kernel/kernel/Networking/RTL8169/RTL8169.cpp @@ -161,9 +161,8 @@ namespace Kernel BAN::ErrorOr RTL8169::enable_interrupt() { - TRY(m_pci_device.reserve_irqs(1)); - set_irq(m_pci_device.get_irq(0)); - Interruptable::enable_interrupt(); + TRY(m_pci_device.reserve_interrupts(1)); + m_pci_device.enable_interrupt(0, *this); m_io_bar_region->write16(RTL8169_IO_IMR, RTL8169_IR_ROK diff --git a/kernel/kernel/PCI.cpp b/kernel/kernel/PCI.cpp index 2c76f43369..c02ca84f19 100644 --- a/kernel/kernel/PCI.cpp +++ b/kernel/kernel/PCI.cpp @@ -175,6 +175,23 @@ namespace Kernel::PCI } } + BAN::Optional PCIManager::reserve_msi() + { + SpinLockGuard _(m_reserved_msi_lock); + + for (uint8_t i = 0; i < m_msi_count; i++) + { + const uint8_t byte = i / 8; + const uint8_t bit = i % 8; + if (m_reserved_msi_bitmap[byte] & (1 << bit)) + continue; + m_reserved_msi_bitmap[byte] |= 1 << bit; + return IRQ_MSI_BASE - IRQ_VECTOR_BASE + i; + } + + return {}; + } + void PCIManager::initialize_devices(bool disable_usb) { for_each_device( @@ -360,63 +377,24 @@ namespace Kernel::PCI } } - BAN::ErrorOr PCI::Device::reserve_irqs(uint8_t count) + uint8_t PCI::Device::get_interrupt(uint8_t index) const { - if (!InterruptController::get().is_using_apic()) + ASSERT(m_offset_msi.has_value() || m_offset_msi_x.has_value() || !InterruptController::get().is_using_apic()); + ASSERT(index < m_reserved_interrupt_count); + + uint8_t count_found = 0; + for (uint32_t irq = 0; irq < 0x100; irq++) { - if (count > 1) - { - dwarnln("PIC: could not allocate {} interrupts, (currently) only {} supported", count, 1); - return BAN::Error::from_errno(EFAULT); - } - enable_pin_interrupts(); - } - else if (m_offset_msi_x.has_value()) - { - uint16_t msg_ctrl = read_word(*m_offset_msi_x + 0x02); - if (count > (msg_ctrl & 0x7FF) + 1) - { - dwarnln("MSI-X: could not allocate {} interrupts, only {} supported", count, (msg_ctrl & 0x7FF) + 1); - return BAN::Error::from_errno(EFAULT); - } - msg_ctrl |= 1 << 15; // Enable - write_word(*m_offset_msi_x + 0x02, msg_ctrl); - disable_pin_interrupts(); - } - else if (m_offset_msi.has_value()) - { - if (count > 1) - { - dwarnln("MSI: could not allocate {} interrupts, (currently) only {} supported", count, 1); - return BAN::Error::from_errno(EFAULT); - } - uint16_t msg_ctrl = read_word(*m_offset_msi + 0x02); - msg_ctrl &= ~(0x07 << 4); // Only one interrupt - msg_ctrl |= 1u << 0; // Enable - write_word(*m_offset_msi + 0x02, msg_ctrl); - disable_pin_interrupts(); - } - else - { - dwarnln("Could not reserve interrupt for PCI device. No MSI, MSI-X or interrupt line is used"); - return BAN::Error::from_errno(EFAULT); + const uint8_t byte = irq / 8; + const uint8_t bit = irq % 8; + + if (m_reserved_interrupts[byte] & (1 << bit)) + count_found++; + if (index + 1 == count_found) + return irq; } - for (; m_reserved_irq_count < count; m_reserved_irq_count++) - { - auto irq = InterruptController::get().get_free_irq(); - if (!irq.has_value()) - { - dwarnln("Could not reserve interrupt for PCI {}:{}.{}", m_bus, m_dev, m_func); - return BAN::Error::from_errno(EFAULT); - } - - ASSERT(*irq < 32); - ASSERT(!(m_reserved_irqs & (1 << *irq))); - m_reserved_irqs |= 1 << *irq; - } - - return {}; + ASSERT_NOT_REACHED(); } static constexpr uint64_t msi_message_address() @@ -429,79 +407,145 @@ namespace Kernel::PCI return (IRQ_VECTOR_BASE + irq) & 0xFF; } - uint8_t PCI::Device::get_irq(uint8_t index) + void PCI::Device::enable_interrupt(uint8_t index, Interruptable& interruptable) { - ASSERT(m_offset_msi.has_value() || m_offset_msi_x.has_value() || !InterruptController::get().is_using_apic()); - ASSERT(index < m_reserved_irq_count); + const uint8_t irq = get_interrupt(index); + interruptable.set_irq(irq); - uint8_t count_found = 0; - uint8_t irq = 0xFF; - for (uint8_t i = 0; i < 32; i++) + switch (m_interrupt_mechanism) { - if (m_reserved_irqs & (1 << i)) - count_found++; - if (count_found > index) + case InterruptMechanism::NONE: + ASSERT_NOT_REACHED(); + case InterruptMechanism::PIN: + enable_pin_interrupts(); + write_byte(PCI_REG_IRQ_LINE, irq); + InterruptController::get().enable_irq(irq); + break; + case InterruptMechanism::MSI: { - irq = i; + disable_pin_interrupts(); + + uint16_t msg_ctrl = read_word(*m_offset_msi + 0x02); + msg_ctrl &= ~(0x07 << 4); // Only one interrupt + msg_ctrl |= 1u << 0; // Enable + write_word(*m_offset_msi + 0x02, msg_ctrl); + + const uint64_t msg_addr = msi_message_address(); + const uint32_t msg_data = msi_message_data(irq); + + if (msg_ctrl & (1 << 7)) + { + write_dword(*m_offset_msi + 0x04, msg_addr & 0xFFFFFFFF); + write_dword(*m_offset_msi + 0x08, msg_addr >> 32); + write_word(*m_offset_msi + 0x12, msg_data); + } + else + { + write_dword(*m_offset_msi + 0x04, msg_addr & 0xFFFFFFFF); + write_word(*m_offset_msi + 0x08, msg_data); + } + + break; + } + case InterruptMechanism::MSIX: + { + disable_pin_interrupts(); + + uint16_t msg_ctrl = read_word(*m_offset_msi_x + 0x02); + msg_ctrl |= 1 << 15; // Enable + write_word(*m_offset_msi_x + 0x02, msg_ctrl); + + const uint32_t dword1 = read_dword(*m_offset_msi_x + 0x04); + const uint32_t offset = dword1 & ~7u; + const uint8_t bir = dword1 & 7u; + + const uint64_t msg_addr = msi_message_address(); + const uint32_t msg_data = msi_message_data(irq); + + auto bar = MUST(allocate_bar_region(bir)); + ASSERT(bar->type() == BarType::MEM); + + auto& msi_x_entry = reinterpret_cast(bar->vaddr() + offset)[index]; + msi_x_entry.msg_addr_low = msg_addr & 0xFFFFFFFF; + msi_x_entry.msg_addr_high = msg_addr >> 32;; + msi_x_entry.msg_data = msg_data; + msi_x_entry.vector_ctrl = msi_x_entry.vector_ctrl & ~1u; + break; } } - ASSERT(irq != 0xFF); + } - // Legacy PIC just uses the interrupt line field - if (!InterruptController::get().is_using_apic()) - { - write_byte(PCI_REG_IRQ_LINE, irq); - return irq; - } + BAN::ErrorOr PCI::Device::reserve_interrupts(uint8_t count) + { + // FIXME: Allow "late" interrupt reserving + ASSERT(m_reserved_interrupt_count == 0); - if (m_offset_msi_x.has_value()) - { - uint32_t dword0 = read_dword(*m_offset_msi_x); - ASSERT((dword0 & 0xFF) == 0x11); - - uint32_t dword1 = read_dword(*m_offset_msi_x + 0x04); - uint32_t offset = dword1 & ~7u; - uint8_t bir = dword1 & 7u; - - uint64_t msg_addr = msi_message_address(); - uint32_t msg_data = msi_message_data(irq); - - auto bar = MUST(allocate_bar_region(bir)); - ASSERT(bar->type() == BarType::MEM); - auto& msi_x_entry = reinterpret_cast(bar->vaddr() + offset)[index]; - msi_x_entry.msg_addr_low = msg_addr & 0xFFFFFFFF; - msi_x_entry.msg_addr_high = msg_addr >> 32;; - msi_x_entry.msg_data = msg_data; - msi_x_entry.vector_ctrl = msi_x_entry.vector_ctrl & ~1u; - - return irq; - } - - if (m_offset_msi.has_value()) - { - uint32_t dword0 = read_dword(*m_offset_msi); - ASSERT((dword0 & 0xFF) == 0x05); - - uint64_t msg_addr = msi_message_address(); - uint32_t msg_data = msi_message_data(irq); - - if (dword0 & (1 << 23)) + const auto mechanism = + [&]() -> InterruptMechanism { - write_dword(*m_offset_msi + 0x04, msg_addr & 0xFFFFFFFF); - write_dword(*m_offset_msi + 0x08, msg_addr >> 32); - write_word(*m_offset_msi + 0x12, msg_data); - } - else - { - write_dword(*m_offset_msi + 0x04, msg_addr & 0xFFFFFFFF); - write_word(*m_offset_msi + 0x08, msg_data); - } + if (!InterruptController::get().is_using_apic()) + { + // FIXME: support multiple PIN interrupts + if (count == 1) + return InterruptMechanism::PIN; - return irq; + // MSI cannot be used without LAPIC + return InterruptMechanism::NONE; + } + + if (m_offset_msi_x.has_value()) + { + const uint16_t msg_ctrl = read_word(*m_offset_msi_x + 0x02); + if (count <= (msg_ctrl & 0x7FF) + 1) + return InterruptMechanism::MSIX; + } + + if (m_offset_msi.has_value()) + { + if (count == 1) + return InterruptMechanism::MSI; + // FIXME: support multiple message + } + + // FIXME: support ioapic + + return InterruptMechanism::NONE; + }(); + + auto get_interrupt_func = + [mechanism]() -> BAN::Optional + { + switch (mechanism) + { + case InterruptMechanism::NONE: + return {}; + case InterruptMechanism::PIN: + return InterruptController::get().get_free_irq(); + case InterruptMechanism::MSI: + case InterruptMechanism::MSIX: + return PCIManager::get().reserve_msi(); + } + ASSERT_NOT_REACHED(); + }; + + for (uint8_t i = 0; i < count; i++) + { + const auto irq = get_interrupt_func(); + if (!irq.has_value()) + { + dwarnln("Could not reserve {} MSI(-X) interrupts", count); + return BAN::Error::from_errno(EFAULT); + } + const uint8_t byte = irq.value() / 8; + const uint8_t bit = irq.value() % 8; + m_reserved_interrupts[byte] |= 1 << bit; } - ASSERT_NOT_REACHED(); + m_interrupt_mechanism = mechanism; + m_reserved_interrupt_count = count; + + return {}; } void PCI::Device::set_command_bits(uint16_t mask) diff --git a/kernel/kernel/Processor.cpp b/kernel/kernel/Processor.cpp index 31524bdace..50b27243ef 100644 --- a/kernel/kernel/Processor.cpp +++ b/kernel/kernel/Processor.cpp @@ -368,7 +368,7 @@ namespace Kernel // NOTE: This is offset by 2 pointers since interrupt without PL change // does not push SP and SS. This allows accessing "whole" interrupt stack. :: [load_sp]"r"(Processor::current_stack_top() - 2 * sizeof(uintptr_t)), - [yield]"i"(IRQ_VECTOR_BASE + IRQ_YIELD) + [yield]"i"(IRQ_YIELD) : "memory", "rcx" ); #elif ARCH(i686) @@ -380,7 +380,7 @@ namespace Kernel // NOTE: This is offset by 2 pointers since interrupt without PL change // does not push SP and SS. This allows accessing "whole" interrupt stack. :: [load_sp]"r"(Processor::current_stack_top() - 2 * sizeof(uintptr_t)), - [yield]"i"(IRQ_VECTOR_BASE + IRQ_YIELD) + [yield]"i"(IRQ_YIELD) : "memory", "ecx" ); #else diff --git a/kernel/kernel/Storage/ATA/AHCI/Controller.cpp b/kernel/kernel/Storage/ATA/AHCI/Controller.cpp index de02323f4c..3b81fa3d45 100644 --- a/kernel/kernel/Storage/ATA/AHCI/Controller.cpp +++ b/kernel/kernel/Storage/ATA/AHCI/Controller.cpp @@ -25,9 +25,8 @@ namespace Kernel // Enable interrupts and bus mastering m_pci_device.enable_bus_mastering(); - TRY(m_pci_device.reserve_irqs(1)); - set_irq(m_pci_device.get_irq(0)); - enable_interrupt(); + TRY(m_pci_device.reserve_interrupts(1)); + m_pci_device.enable_interrupt(0, *this); abar_mem.ghc = abar_mem.ghc | SATA_GHC_INTERRUPT_ENABLE; m_command_slot_count = ((abar_mem.cap >> 8) & 0x1F) + 1; diff --git a/kernel/kernel/Storage/ATA/ATABus.cpp b/kernel/kernel/Storage/ATA/ATABus.cpp index 2b6922f687..643a4685a0 100644 --- a/kernel/kernel/Storage/ATA/ATABus.cpp +++ b/kernel/kernel/Storage/ATA/ATABus.cpp @@ -25,14 +25,13 @@ namespace Kernel return BAN::Error::from_errno(ENODEV); } bus->set_irq(irq); + InterruptController::get().enable_irq(irq); TRY(bus->initialize()); return bus; } BAN::ErrorOr ATABus::initialize() { - enable_interrupt(); - BAN::Vector identify_buffer; MUST(identify_buffer.resize(256)); diff --git a/kernel/kernel/Storage/NVMe/Controller.cpp b/kernel/kernel/Storage/NVMe/Controller.cpp index 50674a502f..68b1a95d40 100644 --- a/kernel/kernel/Storage/NVMe/Controller.cpp +++ b/kernel/kernel/Storage/NVMe/Controller.cpp @@ -82,7 +82,7 @@ namespace Kernel } // One for aq and one for ioq - TRY(m_pci_device.reserve_irqs(2)); + TRY(m_pci_device.reserve_interrupts(2)); auto& cc = m_controller_registers->cc; @@ -237,12 +237,12 @@ namespace Kernel m_controller_registers->acq = completion_queue->paddr(); m_controller_registers->asq = submission_queue->paddr(); - uint8_t irq = m_pci_device.get_irq(0); - dprintln_if(DEBUG_NVMe, " admin queue using irq {}", irq); + dprintln_if(DEBUG_NVMe, " admin queue using irq {}", m_pci_device.get_interrupt(0)); auto& doorbell = *reinterpret_cast(m_bar0->vaddr() + NVMe::ControllerRegisters::SQ0TDBL); - m_admin_queue = TRY(BAN::UniqPtr::create(BAN::move(completion_queue), BAN::move(submission_queue), doorbell, admin_queue_depth, irq)); + m_admin_queue = TRY(BAN::UniqPtr::create(BAN::move(completion_queue), BAN::move(submission_queue), doorbell, admin_queue_depth)); + m_pci_device.enable_interrupt(0, *m_admin_queue); return {}; } @@ -290,14 +290,14 @@ namespace Kernel } } - uint8_t irq = m_pci_device.get_irq(1); - dprintln_if(DEBUG_NVMe, " io queue using irq {}", irq); + dprintln_if(DEBUG_NVMe, " io queue using irq {}", m_pci_device.get_interrupt(1)); const uint32_t doorbell_stride = 1 << (2 + m_controller_registers->cap.dstrd); const uint32_t doorbell_offset = 2 * doorbell_stride; auto& doorbell = *reinterpret_cast(m_bar0->vaddr() + NVMe::ControllerRegisters::SQ0TDBL + doorbell_offset); - m_io_queue = TRY(BAN::UniqPtr::create(BAN::move(completion_queue), BAN::move(submission_queue), doorbell, queue_elems, irq)); + m_io_queue = TRY(BAN::UniqPtr::create(BAN::move(completion_queue), BAN::move(submission_queue), doorbell, queue_elems)); + m_pci_device.enable_interrupt(1, *m_io_queue); return {}; } diff --git a/kernel/kernel/Storage/NVMe/Queue.cpp b/kernel/kernel/Storage/NVMe/Queue.cpp index 3960299a45..7241696f53 100644 --- a/kernel/kernel/Storage/NVMe/Queue.cpp +++ b/kernel/kernel/Storage/NVMe/Queue.cpp @@ -9,7 +9,7 @@ namespace Kernel static constexpr uint64_t s_nvme_command_timeout_ms = 1000; static constexpr uint64_t s_nvme_command_poll_timeout_ms = 20; - NVMeQueue::NVMeQueue(BAN::UniqPtr&& cq, BAN::UniqPtr&& sq, volatile NVMe::DoorbellRegisters& db, uint32_t qdepth, uint8_t irq) + NVMeQueue::NVMeQueue(BAN::UniqPtr&& cq, BAN::UniqPtr&& sq, volatile NVMe::DoorbellRegisters& db, uint32_t qdepth) : m_completion_queue(BAN::move(cq)) , m_submission_queue(BAN::move(sq)) , m_doorbell(db) @@ -17,8 +17,6 @@ namespace Kernel { for (uint32_t i = qdepth; i < m_mask_bits; i++) m_used_mask |= (size_t)1 << i; - set_irq(irq); - enable_interrupt(); } void NVMeQueue::handle_irq() diff --git a/kernel/kernel/Terminal/Serial.cpp b/kernel/kernel/Terminal/Serial.cpp index 5bfdc2fc5c..c6d7febb07 100644 --- a/kernel/kernel/Terminal/Serial.cpp +++ b/kernel/kernel/Terminal/Serial.cpp @@ -188,14 +188,14 @@ namespace Kernel IO::outb(COM1_PORT + 1, 1); TRY(InterruptController::get().reserve_irq(COM1_IRQ)); tty->set_irq(COM1_IRQ); - tty->enable_interrupt(); + InterruptController::get().enable_irq(COM1_IRQ); } else if (serial.port() == COM2_PORT) { IO::outb(COM2_PORT + 1, 1); TRY(InterruptController::get().reserve_irq(COM2_IRQ)); tty->set_irq(COM2_IRQ); - tty->enable_interrupt(); + InterruptController::get().enable_irq(COM2_IRQ); } auto ref_ptr = BAN::RefPtr::adopt(tty); diff --git a/kernel/kernel/Timer/HPET.cpp b/kernel/kernel/Timer/HPET.cpp index 4935163b03..e7cdf2a2cd 100644 --- a/kernel/kernel/Timer/HPET.cpp +++ b/kernel/kernel/Timer/HPET.cpp @@ -223,7 +223,7 @@ namespace Kernel TRY(InterruptController::get().reserve_irq(0)); set_irq(0); - enable_interrupt(); + InterruptController::get().enable_irq(0); return {}; } diff --git a/kernel/kernel/Timer/PIT.cpp b/kernel/kernel/Timer/PIT.cpp index c4009fc66b..50611d9ec7 100644 --- a/kernel/kernel/Timer/PIT.cpp +++ b/kernel/kernel/Timer/PIT.cpp @@ -48,7 +48,7 @@ namespace Kernel MUST(InterruptController::get().reserve_irq(PIT_IRQ)); set_irq(PIT_IRQ); - enable_interrupt(); + InterruptController::get().enable_irq(PIT_IRQ); } void PIT::handle_irq() diff --git a/kernel/kernel/USB/XHCI/Controller.cpp b/kernel/kernel/USB/XHCI/Controller.cpp index 3b71db56a6..f8309c14ff 100644 --- a/kernel/kernel/USB/XHCI/Controller.cpp +++ b/kernel/kernel/USB/XHCI/Controller.cpp @@ -204,7 +204,7 @@ namespace Kernel BAN::ErrorOr XHCIController::initialize_primary_interrupter() { - TRY(m_pci_device.reserve_irqs(1)); + TRY(m_pci_device.reserve_interrupts(1)); auto& runtime = runtime_regs(); @@ -227,8 +227,7 @@ namespace Kernel primary_interrupter.iman = primary_interrupter.iman | XHCI::IMAN::InterruptPending | XHCI::IMAN::InterruptEnable; - set_irq(m_pci_device.get_irq(0)); - enable_interrupt(); + m_pci_device.enable_interrupt(0, *this); return {}; }