From 539afb329a2936b1bf51be56efc246c5c97df90d Mon Sep 17 00:00:00 2001 From: Bananymous Date: Tue, 23 Jul 2024 02:28:52 +0300 Subject: [PATCH] Kernel: All processors use LAPIC timer when running with APIC This makes scheduler preemption much cleaner as bsb does not have to send smp messages to notify other processes about timer interrupt. Also PIT percision is now "full" 0.8 us instead of 1 ms that I was using before. --- kernel/arch/i686/interrupts.S | 17 +++++++ kernel/arch/x86_64/interrupts.S | 7 +++ kernel/include/kernel/APIC.h | 17 ++++--- kernel/include/kernel/IDT.h | 3 +- kernel/include/kernel/Processor.h | 2 - kernel/include/kernel/Scheduler.h | 2 - kernel/include/kernel/Timer/HPET.h | 2 + kernel/include/kernel/Timer/PIT.h | 7 ++- kernel/include/kernel/Timer/Timer.h | 12 +++++ kernel/kernel/APIC.cpp | 58 +++++++++++++++++++--- kernel/kernel/IDT.cpp | 9 ++++ kernel/kernel/Processor.cpp | 14 ++---- kernel/kernel/Scheduler.cpp | 19 +------- kernel/kernel/Timer/HPET.cpp | 36 ++++++++++++-- kernel/kernel/Timer/PIT.cpp | 76 ++++++++++++++++++++++------- kernel/kernel/Timer/Timer.cpp | 9 ++-- kernel/kernel/kernel.cpp | 7 +++ 17 files changed, 224 insertions(+), 73 deletions(-) diff --git a/kernel/arch/i686/interrupts.S b/kernel/arch/i686/interrupts.S index 43b8ee69af..6838ef537f 100644 --- a/kernel/arch/i686/interrupts.S +++ b/kernel/arch/i686/interrupts.S @@ -113,6 +113,23 @@ asm_ipi_handler: pop_userspace iret + +.global asm_timer_handler +asm_timer_handler: + push_userspace + load_kernel_segments + + movl %esp, %ebp + subl $15, %esp + andl $0xFFFFFFF0, %esp + + call cpp_timer_handler + + movl %ebp, %esp + + pop_userspace + iret + .macro isr n .global isr\n isr\n: diff --git a/kernel/arch/x86_64/interrupts.S b/kernel/arch/x86_64/interrupts.S index ca9fb50b9c..35862f758a 100644 --- a/kernel/arch/x86_64/interrupts.S +++ b/kernel/arch/x86_64/interrupts.S @@ -77,6 +77,13 @@ asm_ipi_handler: popaq iretq +.global asm_timer_handler +asm_timer_handler: + pushaq + call cpp_timer_handler + popaq + iretq + .macro isr n .global isr\n isr\n: diff --git a/kernel/include/kernel/APIC.h b/kernel/include/kernel/APIC.h index b0bd0427b7..5a7e9e1bbf 100644 --- a/kernel/include/kernel/APIC.h +++ b/kernel/include/kernel/APIC.h @@ -23,6 +23,8 @@ namespace Kernel virtual void broadcast_ipi() override; virtual void enable() override; + void initialize_timer(); + private: uint32_t read_from_local_apic(ptrdiff_t); void write_to_local_apic(ptrdiff_t, uint32_t); @@ -58,13 +60,14 @@ namespace Kernel }; private: - SpinLock m_lock; - BAN::Vector m_processors; - Kernel::paddr_t m_local_apic_paddr = 0; - Kernel::vaddr_t m_local_apic_vaddr = 0; - BAN::Vector m_io_apics; - uint8_t m_irq_overrides[0x100] {}; - uint8_t m_reserved_gsis[0x100 / 8] {}; + SpinLock m_lock; + BAN::Vector m_processors; + Kernel::paddr_t m_local_apic_paddr = 0; + Kernel::vaddr_t m_local_apic_vaddr = 0; + BAN::Vector m_io_apics; + uint8_t m_irq_overrides[0x100] {}; + uint8_t m_reserved_gsis[0x100 / 8] {}; + uint64_t m_lapic_timer_frequency_hz { 0 }; }; } diff --git a/kernel/include/kernel/IDT.h b/kernel/include/kernel/IDT.h index 921819e990..a3cf7506cc 100644 --- a/kernel/include/kernel/IDT.h +++ b/kernel/include/kernel/IDT.h @@ -7,9 +7,10 @@ #include -constexpr uint8_t IRQ_VECTOR_BASE = 0x20; +constexpr uint8_t IRQ_VECTOR_BASE = 0x20; constexpr uint8_t IRQ_YIELD = 32; constexpr uint8_t IRQ_IPI = 33; +constexpr uint8_t IRQ_TIMER = 34; namespace Kernel { diff --git a/kernel/include/kernel/Processor.h b/kernel/include/kernel/Processor.h index ea87fe94be..e64e974cbb 100644 --- a/kernel/include/kernel/Processor.h +++ b/kernel/include/kernel/Processor.h @@ -56,8 +56,6 @@ namespace Kernel FlushTLB, NewThread, UnblockThread, - // FIXME: all processors should LAPIC for their preemption - SchedulerPreemption, }; SMPMessage* next { nullptr }; Type type; diff --git a/kernel/include/kernel/Scheduler.h b/kernel/include/kernel/Scheduler.h index 3b4a8644de..90e0efdca8 100644 --- a/kernel/include/kernel/Scheduler.h +++ b/kernel/include/kernel/Scheduler.h @@ -110,8 +110,6 @@ namespace Kernel class ProcessorID find_least_loaded_processor() const; - void preempt(); - void handle_unblock_request(const UnblockRequest&); void handle_new_thread_request(const NewThreadRequest&); diff --git a/kernel/include/kernel/Timer/HPET.h b/kernel/include/kernel/Timer/HPET.h index 4b35b02436..d6f61f5801 100644 --- a/kernel/include/kernel/Timer/HPET.h +++ b/kernel/include/kernel/Timer/HPET.h @@ -19,6 +19,8 @@ namespace Kernel virtual uint64_t ns_since_boot() const override; virtual timespec time_since_boot() const override; + virtual void pre_scheduler_sleep_ns(uint64_t) override; + virtual void handle_irq() override; private: diff --git a/kernel/include/kernel/Timer/PIT.h b/kernel/include/kernel/Timer/PIT.h index 26b7f7329c..96d991782e 100644 --- a/kernel/include/kernel/Timer/PIT.h +++ b/kernel/include/kernel/Timer/PIT.h @@ -16,15 +16,18 @@ namespace Kernel virtual uint64_t ns_since_boot() const override; virtual timespec time_since_boot() const override; + virtual void pre_scheduler_sleep_ns(uint64_t) override; + virtual void handle_irq() override; private: void initialize(); - uint64_t read_counter() const; + + uint64_t read_counter_ns() const; private: mutable SpinLock m_lock; - uint64_t m_system_time { 0 }; + uint64_t m_system_time_ms { 0 }; }; } diff --git a/kernel/include/kernel/Timer/Timer.h b/kernel/include/kernel/Timer/Timer.h index 941febd0e2..65b6134352 100644 --- a/kernel/include/kernel/Timer/Timer.h +++ b/kernel/include/kernel/Timer/Timer.h @@ -16,6 +16,16 @@ namespace Kernel virtual uint64_t ms_since_boot() const = 0; virtual uint64_t ns_since_boot() const = 0; virtual timespec time_since_boot() const = 0; + + virtual void pre_scheduler_sleep_ns(uint64_t) = 0; + + void dont_invoke_scheduler() { m_should_invoke_scheduler = false; } + + protected: + bool should_invoke_scheduler() const { return m_should_invoke_scheduler; } + + private: + bool m_should_invoke_scheduler { true }; }; class SystemTimer : public Timer @@ -29,6 +39,8 @@ namespace Kernel virtual uint64_t ns_since_boot() const override; virtual timespec time_since_boot() const override; + virtual void pre_scheduler_sleep_ns(uint64_t) override; + void sleep_ms(uint64_t ms) const { return sleep_ns(ms * 1'000'000); } void sleep_ns(uint64_t ns) const; diff --git a/kernel/kernel/APIC.cpp b/kernel/kernel/APIC.cpp index d3f0e425f6..ac393a1b4c 100644 --- a/kernel/kernel/APIC.cpp +++ b/kernel/kernel/APIC.cpp @@ -17,6 +17,11 @@ #define LAPIC_ICR_LO_REG 0x300 #define LAPIC_ICR_HI_REG 0x310 +#define LAPIC_TIMER_LVT 0x320 +#define LAPIC_TIMER_INITIAL_REG 0x380 +#define LAPIC_TIMER_CURRENT_REG 0x390 +#define LAPIC_TIMER_DIVIDE_REG 0x3E0 + #define IOAPIC_MAX_REDIRS 0x01 #define IOAPIC_REDIRS 0x10 @@ -60,6 +65,27 @@ namespace Kernel ICR_LO_destination_shorthand_all_excluding_self = 0b11 << 18, }; + enum TimerLVT : uint32_t + { + TimerModeOneShot = 0b00 << 17, + TimerModePeriodic = 0b01 << 17, + TimerModeTSCDeadline = 0b10 << 17, + + TimerMask = 1 << 16, + }; + + enum TimerDivideRegister : uint32_t + { + DivideBy2 = 0b0000, + DivideBy4 = 0b0001, + DivideBy8 = 0b0010, + DivideBy16 = 0b0011, + DivideBy32 = 0b1000, + DivideBy64 = 0b1001, + DivideBy128 = 0b1010, + DivideBy1 = 0b1011, + }; + struct MADT : public Kernel::ACPI::SDTHeader { uint32_t local_apic; @@ -223,12 +249,7 @@ namespace Kernel void APIC::initialize_multiprocessor() { - constexpr auto udelay = - [](uint64_t us) { - uint64_t wake_time = SystemTimer::get().ns_since_boot() + us * 1000; - while (SystemTimer::get().ns_since_boot() < wake_time) - __builtin_ia32_pause(); - }; + constexpr auto udelay = [](uint64_t us) { SystemTimer::get().pre_scheduler_sleep_ns(us * 1000); }; const auto send_ipi = [&](uint8_t processor, uint32_t data, uint64_t ud) @@ -367,6 +388,31 @@ namespace Kernel void APIC::enable() { write_to_local_apic(LAPIC_SIV_REG, read_from_local_apic(LAPIC_SIV_REG) | 0x1FF); + initialize_timer(); + } + + static SpinLock s_timer_init_lock; + + void APIC::initialize_timer() + { + { + constexpr uint64_t measuring_duration_ms = 100; + + SpinLockGuard _(s_timer_init_lock); + + write_to_local_apic(LAPIC_TIMER_LVT, TimerModeOneShot | TimerMask); + write_to_local_apic(LAPIC_TIMER_DIVIDE_REG, DivideBy2); + write_to_local_apic(LAPIC_TIMER_INITIAL_REG, 0xFFFFFFFF); + SystemTimer::get().pre_scheduler_sleep_ns(measuring_duration_ms * 1'000'000); + + const uint32_t counter = read_from_local_apic(LAPIC_TIMER_CURRENT_REG); + m_lapic_timer_frequency_hz = static_cast(0xFFFFFFFF - counter) * 2 * (1000 / measuring_duration_ms); + + dprintln("CPU {}: lapic timer frequency: {} Hz", Kernel::Processor::current_id(), m_lapic_timer_frequency_hz); + } + + write_to_local_apic(LAPIC_TIMER_LVT, TimerModePeriodic | (IRQ_VECTOR_BASE + IRQ_TIMER)); + write_to_local_apic(LAPIC_TIMER_INITIAL_REG, m_lapic_timer_frequency_hz / 2 / 100); } uint32_t APIC::read_from_local_apic(ptrdiff_t offset) diff --git a/kernel/kernel/IDT.cpp b/kernel/kernel/IDT.cpp index c200e3d1db..c4155d2242 100644 --- a/kernel/kernel/IDT.cpp +++ b/kernel/kernel/IDT.cpp @@ -333,6 +333,13 @@ done: Processor::handle_ipi(); } + extern "C" void cpp_timer_handler() + { + ASSERT(InterruptController::get().is_in_service(IRQ_TIMER)); + InterruptController::get().eoi(IRQ_TIMER); + Processor::scheduler().timer_interrupt(); + } + extern "C" void cpp_irq_handler(uint32_t irq) { if (g_paniced) @@ -409,6 +416,7 @@ done: extern "C" void asm_yield_handler(); extern "C" void asm_ipi_handler(); + extern "C" void asm_timer_handler(); extern "C" void asm_syscall_handler(); IDT* IDT::create() @@ -428,6 +436,7 @@ done: idt->register_interrupt_handler(IRQ_VECTOR_BASE + IRQ_YIELD, asm_yield_handler); idt->register_interrupt_handler(IRQ_VECTOR_BASE + IRQ_IPI, asm_ipi_handler); + idt->register_interrupt_handler(IRQ_VECTOR_BASE + IRQ_TIMER, asm_timer_handler); idt->register_syscall_handler(0x80, asm_syscall_handler); diff --git a/kernel/kernel/Processor.cpp b/kernel/kernel/Processor.cpp index 2c40a1150f..cd0cf73115 100644 --- a/kernel/kernel/Processor.cpp +++ b/kernel/kernel/Processor.cpp @@ -20,7 +20,7 @@ namespace Kernel static BAN::Atomic s_processors_created { 0 }; // 32 bit milli seconds are definitely enough as APs start on boot - static BAN::Atomic s_first_ap_ready_ms { 0 }; + static BAN::Atomic s_first_ap_ready_ms { static_cast(-1) }; static BAN::Array s_processors; static BAN::Array s_processor_ids { PROCESSOR_NONE }; @@ -130,7 +130,7 @@ namespace Kernel // wait until first AP is ready const uint64_t timeout_ms = SystemTimer::get().ms_since_boot() + 1000; - while (s_first_ap_ready_ms == 0) + while (s_first_ap_ready_ms == static_cast(-1)) { if (SystemTimer::get().ms_since_boot() >= timeout_ms) { @@ -150,7 +150,7 @@ namespace Kernel ASSERT(s_processor_ids[lookup_index] == PROCESSOR_NONE); s_processor_ids[lookup_index] = current_id(); - uint32_t expected = 0; + uint32_t expected = static_cast(-1); s_first_ap_ready_ms.compare_exchange(expected, SystemTimer::get().ms_since_boot()); } @@ -209,8 +209,6 @@ namespace Kernel } ); - bool should_preempt = false; - if (pending) { // reverse smp message queue from LIFO to FIFO @@ -245,9 +243,6 @@ namespace Kernel case SMPMessage::Type::UnblockThread: processor.m_scheduler->handle_unblock_request(message->unblock_thread); break; - case SMPMessage::Type::SchedulerPreemption: - should_preempt = true; - break; } last_handled = message; @@ -262,9 +257,6 @@ namespace Kernel ); } - if (should_preempt) - processor.m_scheduler->preempt(); - set_interrupt_state(state); } diff --git a/kernel/kernel/Scheduler.cpp b/kernel/kernel/Scheduler.cpp index 445072bf90..a0db820e33 100644 --- a/kernel/kernel/Scheduler.cpp +++ b/kernel/kernel/Scheduler.cpp @@ -294,7 +294,7 @@ namespace Kernel Processor::yield(); } - void Scheduler::preempt() + void Scheduler::timer_interrupt() { ASSERT(Processor::get_interrupt_state() == InterruptState::Disabled); @@ -321,23 +321,6 @@ namespace Kernel } } - void Scheduler::timer_interrupt() - { - ASSERT(Processor::get_interrupt_state() == InterruptState::Disabled); - - // FIXME: all processors should LAPIC for their preemption - if (Processor::is_smp_enabled()) - { - ASSERT(Processor::current_is_bsb()); - Processor::broadcast_smp_message({ - .type = Processor::SMPMessage::Type::SchedulerPreemption, - .scheduler_preemption = 0 // dummy value - }); - } - - preempt(); - } - void Scheduler::handle_unblock_request(const UnblockRequest& request) { ASSERT(Processor::get_interrupt_state() == InterruptState::Disabled); diff --git a/kernel/kernel/Timer/HPET.cpp b/kernel/kernel/Timer/HPET.cpp index c5b8e05dd8..4935163b03 100644 --- a/kernel/kernel/Timer/HPET.cpp +++ b/kernel/kernel/Timer/HPET.cpp @@ -254,9 +254,9 @@ namespace Kernel void HPET::handle_irq() { - auto& regs = registers(); - { + auto& regs = registers(); + SpinLockGuard _(m_lock); uint64_t current_ticks; @@ -272,7 +272,8 @@ namespace Kernel m_last_ticks = current_ticks; } - Processor::scheduler().timer_interrupt(); + if (should_invoke_scheduler()) + Processor::scheduler().timer_interrupt(); } uint64_t HPET::ms_since_boot() const @@ -303,4 +304,33 @@ namespace Kernel }; } + void HPET::pre_scheduler_sleep_ns(uint64_t ns) + { + auto& regs = registers(); + + const uint64_t target_ticks = BAN::Math::div_round_up(ns * FS_PER_NS, regs.counter_clk_period); + + if (m_is_64bit) + { + const uint64_t target_counter = regs.main_counter.full + target_ticks; + while (regs.main_counter.full < target_counter) + __builtin_ia32_pause(); + } + else + { + uint64_t elapsed_ticks = 0; + uint64_t last_counter = regs.main_counter.low; + while (elapsed_ticks < target_ticks) + { + const uint64_t current_counter = regs.main_counter.low; + if (last_counter <= current_counter) + elapsed_ticks += current_counter - last_counter; + else + elapsed_ticks += 0xFFFFFFFF + current_counter - last_counter; + last_counter = current_counter; + __builtin_ia32_pause(); + } + } + } + } diff --git a/kernel/kernel/Timer/PIT.cpp b/kernel/kernel/Timer/PIT.cpp index ec15b5f102..c4009fc66b 100644 --- a/kernel/kernel/Timer/PIT.cpp +++ b/kernel/kernel/Timer/PIT.cpp @@ -18,17 +18,19 @@ #define ACCESS_HI 0x10 #define ACCESS_LO 0x20 -#define MODE_SQUARE_WAVE 0x06 +#define MODE_RATE_GENERATOR 0x05 #define BASE_FREQUENCY 1193182 -#define TICKS_PER_SECOND 1000 #define MS_PER_S 1'000 +#define NS_PER_MS 1'000'000 #define NS_PER_S 1'000'000'000 namespace Kernel { + constexpr uint16_t s_ticks_per_ms = BASE_FREQUENCY / 1000; + BAN::ErrorOr> PIT::create() { PIT* pit = new PIT(); @@ -40,12 +42,9 @@ namespace Kernel void PIT::initialize() { - constexpr uint16_t timer_reload = BASE_FREQUENCY / TICKS_PER_SECOND; - - IO::outb(PIT_CTL, SELECT_CHANNEL0 | ACCESS_LO | ACCESS_HI | MODE_SQUARE_WAVE); - - IO::outb(TIMER0_CTL, (timer_reload >> 0) & 0xff); - IO::outb(TIMER0_CTL, (timer_reload >> 8) & 0xff); + IO::outb(PIT_CTL, SELECT_CHANNEL0 | ACCESS_LO | ACCESS_HI | MODE_RATE_GENERATOR); + IO::outb(TIMER0_CTL, (s_ticks_per_ms >> 0) & 0xff); + IO::outb(TIMER0_CTL, (s_ticks_per_ms >> 8) & 0xff); MUST(InterruptController::get().reserve_irq(PIT_IRQ)); set_irq(PIT_IRQ); @@ -56,34 +55,77 @@ namespace Kernel { { SpinLockGuard _(m_lock); - m_system_time++; + m_system_time_ms++; } - Processor::scheduler().timer_interrupt(); + + if (should_invoke_scheduler()) + Processor::scheduler().timer_interrupt(); } - uint64_t PIT::read_counter() const + uint64_t PIT::read_counter_ns() const { SpinLockGuard _(m_lock); - return m_system_time; + + // send latch command + IO::outb(PIT_CTL, SELECT_CHANNEL0); + + // read ticks + uint64_t ticks_this_ms { 0 }; + ticks_this_ms |= IO::inb(TIMER0_CTL); + ticks_this_ms |= IO::inb(TIMER0_CTL) << 8; + ticks_this_ms = s_ticks_per_ms - ticks_this_ms; + + const uint64_t ns_this_ms = ticks_this_ms * NS_PER_S / BASE_FREQUENCY; + return (m_system_time_ms * NS_PER_MS) + ns_this_ms; } uint64_t PIT::ms_since_boot() const { - return read_counter() * (MS_PER_S / TICKS_PER_SECOND); + return read_counter_ns() / NS_PER_MS; } uint64_t PIT::ns_since_boot() const { - return read_counter() * (NS_PER_S / TICKS_PER_SECOND); + return read_counter_ns(); } timespec PIT::time_since_boot() const { - uint64_t ticks = read_counter(); + uint64_t ns = read_counter_ns(); return timespec { - .tv_sec = ticks / TICKS_PER_SECOND, - .tv_nsec = (long)((ticks % TICKS_PER_SECOND) * (NS_PER_S / TICKS_PER_SECOND)) + .tv_sec = static_cast(ns / NS_PER_S), + .tv_nsec = static_cast(ns % NS_PER_S) }; } + void PIT::pre_scheduler_sleep_ns(uint64_t ns) + { + const uint64_t target_ticks = BAN::Math::div_round_up(ns * BASE_FREQUENCY, NS_PER_S); + + SpinLockGuard _(m_lock); + + IO::outb(PIT_CTL, SELECT_CHANNEL0 | ACCESS_LO | ACCESS_HI | MODE_RATE_GENERATOR); + IO::outb(TIMER0_CTL, 0); + IO::outb(TIMER0_CTL, 0); + + IO::outb(PIT_CTL, SELECT_CHANNEL0 | ACCESS_LO | MODE_RATE_GENERATOR); + IO::outb(TIMER0_CTL, 0xFF); + + uint64_t elapsed_ticks = 0; + uint8_t last_ticks = IO::inb(TIMER0_CTL); + while (elapsed_ticks < target_ticks) + { + const uint8_t current_ticks = IO::inb(TIMER0_CTL); + if (last_ticks <= current_ticks) + elapsed_ticks += current_ticks - last_ticks; + else + elapsed_ticks += 0xFF + current_ticks - last_ticks; + last_ticks = current_ticks; + } + + IO::outb(PIT_CTL, SELECT_CHANNEL0 | ACCESS_LO | ACCESS_HI | MODE_RATE_GENERATOR); + IO::outb(TIMER0_CTL, (s_ticks_per_ms >> 0) & 0xFF); + IO::outb(TIMER0_CTL, (s_ticks_per_ms >> 8) & 0xFF); + } + } diff --git a/kernel/kernel/Timer/Timer.cpp b/kernel/kernel/Timer/Timer.cpp index 1c3186f6f1..d4075bdfc9 100644 --- a/kernel/kernel/Timer/Timer.cpp +++ b/kernel/kernel/Timer/Timer.cpp @@ -69,6 +69,11 @@ namespace Kernel return m_timer->time_since_boot(); } + void SystemTimer::pre_scheduler_sleep_ns(uint64_t ns) + { + return m_timer->pre_scheduler_sleep_ns(ns); + } + void SystemTimer::sleep_ns(uint64_t ns) const { if (ns == 0) @@ -76,10 +81,6 @@ namespace Kernel const uint64_t wake_time_ns = ns_since_boot() + ns; Processor::scheduler().block_current_thread(nullptr, wake_time_ns); - - //const uint64_t current_time_ns = ns_since_boot(); - //if (current_time_ns < wake_time_ns) - // dwarnln("sleep woke {} ms too soon", BAN::Math::div_round_up(wake_time_ns - current_time_ns, 1'000'000)); } timespec SystemTimer::real_time() const diff --git a/kernel/kernel/kernel.cpp b/kernel/kernel/kernel.cpp index 753c32b899..3e84b9f164 100644 --- a/kernel/kernel/kernel.cpp +++ b/kernel/kernel/kernel.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -167,6 +168,12 @@ extern "C" void kernel_main(uint32_t boot_magic, uint32_t boot_info) Random::initialize(); dprintln("RNG initialized"); + if (InterruptController::get().is_using_apic()) + { + SystemTimer::get().dont_invoke_scheduler(); + static_cast(InterruptController::get()).initialize_timer(); + } + Processor::wait_until_processors_ready(); MUST(Processor::scheduler().initialize());