diff --git a/kernel/arch/i686/interrupts.S b/kernel/arch/i686/interrupts.S index 43b8ee69af..6838ef537f 100644 --- a/kernel/arch/i686/interrupts.S +++ b/kernel/arch/i686/interrupts.S @@ -113,6 +113,23 @@ asm_ipi_handler: pop_userspace iret + +.global asm_timer_handler +asm_timer_handler: + push_userspace + load_kernel_segments + + movl %esp, %ebp + subl $15, %esp + andl $0xFFFFFFF0, %esp + + call cpp_timer_handler + + movl %ebp, %esp + + pop_userspace + iret + .macro isr n .global isr\n isr\n: diff --git a/kernel/arch/x86_64/interrupts.S b/kernel/arch/x86_64/interrupts.S index ca9fb50b9c..35862f758a 100644 --- a/kernel/arch/x86_64/interrupts.S +++ b/kernel/arch/x86_64/interrupts.S @@ -77,6 +77,13 @@ asm_ipi_handler: popaq iretq +.global asm_timer_handler +asm_timer_handler: + pushaq + call cpp_timer_handler + popaq + iretq + .macro isr n .global isr\n isr\n: diff --git a/kernel/include/kernel/APIC.h b/kernel/include/kernel/APIC.h index b0bd0427b7..5a7e9e1bbf 100644 --- a/kernel/include/kernel/APIC.h +++ b/kernel/include/kernel/APIC.h @@ -23,6 +23,8 @@ namespace Kernel virtual void broadcast_ipi() override; virtual void enable() override; + void initialize_timer(); + private: uint32_t read_from_local_apic(ptrdiff_t); void write_to_local_apic(ptrdiff_t, uint32_t); @@ -58,13 +60,14 @@ namespace Kernel }; private: - SpinLock m_lock; - BAN::Vector m_processors; - Kernel::paddr_t m_local_apic_paddr = 0; - Kernel::vaddr_t m_local_apic_vaddr = 0; - BAN::Vector m_io_apics; - uint8_t m_irq_overrides[0x100] {}; - uint8_t m_reserved_gsis[0x100 / 8] {}; + SpinLock m_lock; + BAN::Vector m_processors; + Kernel::paddr_t m_local_apic_paddr = 0; + Kernel::vaddr_t m_local_apic_vaddr = 0; + BAN::Vector m_io_apics; + uint8_t m_irq_overrides[0x100] {}; + uint8_t m_reserved_gsis[0x100 / 8] {}; + uint64_t m_lapic_timer_frequency_hz { 0 }; }; } diff --git a/kernel/include/kernel/IDT.h b/kernel/include/kernel/IDT.h index 921819e990..a3cf7506cc 100644 --- a/kernel/include/kernel/IDT.h +++ b/kernel/include/kernel/IDT.h @@ -7,9 +7,10 @@ #include -constexpr uint8_t IRQ_VECTOR_BASE = 0x20; +constexpr uint8_t IRQ_VECTOR_BASE = 0x20; constexpr uint8_t IRQ_YIELD = 32; constexpr uint8_t IRQ_IPI = 33; +constexpr uint8_t IRQ_TIMER = 34; namespace Kernel { diff --git a/kernel/include/kernel/Processor.h b/kernel/include/kernel/Processor.h index ea87fe94be..e64e974cbb 100644 --- a/kernel/include/kernel/Processor.h +++ b/kernel/include/kernel/Processor.h @@ -56,8 +56,6 @@ namespace Kernel FlushTLB, NewThread, UnblockThread, - // FIXME: all processors should LAPIC for their preemption - SchedulerPreemption, }; SMPMessage* next { nullptr }; Type type; diff --git a/kernel/include/kernel/Scheduler.h b/kernel/include/kernel/Scheduler.h index 3b4a8644de..90e0efdca8 100644 --- a/kernel/include/kernel/Scheduler.h +++ b/kernel/include/kernel/Scheduler.h @@ -110,8 +110,6 @@ namespace Kernel class ProcessorID find_least_loaded_processor() const; - void preempt(); - void handle_unblock_request(const UnblockRequest&); void handle_new_thread_request(const NewThreadRequest&); diff --git a/kernel/include/kernel/Timer/HPET.h b/kernel/include/kernel/Timer/HPET.h index 4b35b02436..d6f61f5801 100644 --- a/kernel/include/kernel/Timer/HPET.h +++ b/kernel/include/kernel/Timer/HPET.h @@ -19,6 +19,8 @@ namespace Kernel virtual uint64_t ns_since_boot() const override; virtual timespec time_since_boot() const override; + virtual void pre_scheduler_sleep_ns(uint64_t) override; + virtual void handle_irq() override; private: diff --git a/kernel/include/kernel/Timer/PIT.h b/kernel/include/kernel/Timer/PIT.h index 26b7f7329c..96d991782e 100644 --- a/kernel/include/kernel/Timer/PIT.h +++ b/kernel/include/kernel/Timer/PIT.h @@ -16,15 +16,18 @@ namespace Kernel virtual uint64_t ns_since_boot() const override; virtual timespec time_since_boot() const override; + virtual void pre_scheduler_sleep_ns(uint64_t) override; + virtual void handle_irq() override; private: void initialize(); - uint64_t read_counter() const; + + uint64_t read_counter_ns() const; private: mutable SpinLock m_lock; - uint64_t m_system_time { 0 }; + uint64_t m_system_time_ms { 0 }; }; } diff --git a/kernel/include/kernel/Timer/Timer.h b/kernel/include/kernel/Timer/Timer.h index 941febd0e2..65b6134352 100644 --- a/kernel/include/kernel/Timer/Timer.h +++ b/kernel/include/kernel/Timer/Timer.h @@ -16,6 +16,16 @@ namespace Kernel virtual uint64_t ms_since_boot() const = 0; virtual uint64_t ns_since_boot() const = 0; virtual timespec time_since_boot() const = 0; + + virtual void pre_scheduler_sleep_ns(uint64_t) = 0; + + void dont_invoke_scheduler() { m_should_invoke_scheduler = false; } + + protected: + bool should_invoke_scheduler() const { return m_should_invoke_scheduler; } + + private: + bool m_should_invoke_scheduler { true }; }; class SystemTimer : public Timer @@ -29,6 +39,8 @@ namespace Kernel virtual uint64_t ns_since_boot() const override; virtual timespec time_since_boot() const override; + virtual void pre_scheduler_sleep_ns(uint64_t) override; + void sleep_ms(uint64_t ms) const { return sleep_ns(ms * 1'000'000); } void sleep_ns(uint64_t ns) const; diff --git a/kernel/kernel/APIC.cpp b/kernel/kernel/APIC.cpp index d3f0e425f6..ac393a1b4c 100644 --- a/kernel/kernel/APIC.cpp +++ b/kernel/kernel/APIC.cpp @@ -17,6 +17,11 @@ #define LAPIC_ICR_LO_REG 0x300 #define LAPIC_ICR_HI_REG 0x310 +#define LAPIC_TIMER_LVT 0x320 +#define LAPIC_TIMER_INITIAL_REG 0x380 +#define LAPIC_TIMER_CURRENT_REG 0x390 +#define LAPIC_TIMER_DIVIDE_REG 0x3E0 + #define IOAPIC_MAX_REDIRS 0x01 #define IOAPIC_REDIRS 0x10 @@ -60,6 +65,27 @@ namespace Kernel ICR_LO_destination_shorthand_all_excluding_self = 0b11 << 18, }; + enum TimerLVT : uint32_t + { + TimerModeOneShot = 0b00 << 17, + TimerModePeriodic = 0b01 << 17, + TimerModeTSCDeadline = 0b10 << 17, + + TimerMask = 1 << 16, + }; + + enum TimerDivideRegister : uint32_t + { + DivideBy2 = 0b0000, + DivideBy4 = 0b0001, + DivideBy8 = 0b0010, + DivideBy16 = 0b0011, + DivideBy32 = 0b1000, + DivideBy64 = 0b1001, + DivideBy128 = 0b1010, + DivideBy1 = 0b1011, + }; + struct MADT : public Kernel::ACPI::SDTHeader { uint32_t local_apic; @@ -223,12 +249,7 @@ namespace Kernel void APIC::initialize_multiprocessor() { - constexpr auto udelay = - [](uint64_t us) { - uint64_t wake_time = SystemTimer::get().ns_since_boot() + us * 1000; - while (SystemTimer::get().ns_since_boot() < wake_time) - __builtin_ia32_pause(); - }; + constexpr auto udelay = [](uint64_t us) { SystemTimer::get().pre_scheduler_sleep_ns(us * 1000); }; const auto send_ipi = [&](uint8_t processor, uint32_t data, uint64_t ud) @@ -367,6 +388,31 @@ namespace Kernel void APIC::enable() { write_to_local_apic(LAPIC_SIV_REG, read_from_local_apic(LAPIC_SIV_REG) | 0x1FF); + initialize_timer(); + } + + static SpinLock s_timer_init_lock; + + void APIC::initialize_timer() + { + { + constexpr uint64_t measuring_duration_ms = 100; + + SpinLockGuard _(s_timer_init_lock); + + write_to_local_apic(LAPIC_TIMER_LVT, TimerModeOneShot | TimerMask); + write_to_local_apic(LAPIC_TIMER_DIVIDE_REG, DivideBy2); + write_to_local_apic(LAPIC_TIMER_INITIAL_REG, 0xFFFFFFFF); + SystemTimer::get().pre_scheduler_sleep_ns(measuring_duration_ms * 1'000'000); + + const uint32_t counter = read_from_local_apic(LAPIC_TIMER_CURRENT_REG); + m_lapic_timer_frequency_hz = static_cast(0xFFFFFFFF - counter) * 2 * (1000 / measuring_duration_ms); + + dprintln("CPU {}: lapic timer frequency: {} Hz", Kernel::Processor::current_id(), m_lapic_timer_frequency_hz); + } + + write_to_local_apic(LAPIC_TIMER_LVT, TimerModePeriodic | (IRQ_VECTOR_BASE + IRQ_TIMER)); + write_to_local_apic(LAPIC_TIMER_INITIAL_REG, m_lapic_timer_frequency_hz / 2 / 100); } uint32_t APIC::read_from_local_apic(ptrdiff_t offset) diff --git a/kernel/kernel/IDT.cpp b/kernel/kernel/IDT.cpp index c200e3d1db..c4155d2242 100644 --- a/kernel/kernel/IDT.cpp +++ b/kernel/kernel/IDT.cpp @@ -333,6 +333,13 @@ done: Processor::handle_ipi(); } + extern "C" void cpp_timer_handler() + { + ASSERT(InterruptController::get().is_in_service(IRQ_TIMER)); + InterruptController::get().eoi(IRQ_TIMER); + Processor::scheduler().timer_interrupt(); + } + extern "C" void cpp_irq_handler(uint32_t irq) { if (g_paniced) @@ -409,6 +416,7 @@ done: extern "C" void asm_yield_handler(); extern "C" void asm_ipi_handler(); + extern "C" void asm_timer_handler(); extern "C" void asm_syscall_handler(); IDT* IDT::create() @@ -428,6 +436,7 @@ done: idt->register_interrupt_handler(IRQ_VECTOR_BASE + IRQ_YIELD, asm_yield_handler); idt->register_interrupt_handler(IRQ_VECTOR_BASE + IRQ_IPI, asm_ipi_handler); + idt->register_interrupt_handler(IRQ_VECTOR_BASE + IRQ_TIMER, asm_timer_handler); idt->register_syscall_handler(0x80, asm_syscall_handler); diff --git a/kernel/kernel/Processor.cpp b/kernel/kernel/Processor.cpp index 2c40a1150f..cd0cf73115 100644 --- a/kernel/kernel/Processor.cpp +++ b/kernel/kernel/Processor.cpp @@ -20,7 +20,7 @@ namespace Kernel static BAN::Atomic s_processors_created { 0 }; // 32 bit milli seconds are definitely enough as APs start on boot - static BAN::Atomic s_first_ap_ready_ms { 0 }; + static BAN::Atomic s_first_ap_ready_ms { static_cast(-1) }; static BAN::Array s_processors; static BAN::Array s_processor_ids { PROCESSOR_NONE }; @@ -130,7 +130,7 @@ namespace Kernel // wait until first AP is ready const uint64_t timeout_ms = SystemTimer::get().ms_since_boot() + 1000; - while (s_first_ap_ready_ms == 0) + while (s_first_ap_ready_ms == static_cast(-1)) { if (SystemTimer::get().ms_since_boot() >= timeout_ms) { @@ -150,7 +150,7 @@ namespace Kernel ASSERT(s_processor_ids[lookup_index] == PROCESSOR_NONE); s_processor_ids[lookup_index] = current_id(); - uint32_t expected = 0; + uint32_t expected = static_cast(-1); s_first_ap_ready_ms.compare_exchange(expected, SystemTimer::get().ms_since_boot()); } @@ -209,8 +209,6 @@ namespace Kernel } ); - bool should_preempt = false; - if (pending) { // reverse smp message queue from LIFO to FIFO @@ -245,9 +243,6 @@ namespace Kernel case SMPMessage::Type::UnblockThread: processor.m_scheduler->handle_unblock_request(message->unblock_thread); break; - case SMPMessage::Type::SchedulerPreemption: - should_preempt = true; - break; } last_handled = message; @@ -262,9 +257,6 @@ namespace Kernel ); } - if (should_preempt) - processor.m_scheduler->preempt(); - set_interrupt_state(state); } diff --git a/kernel/kernel/Scheduler.cpp b/kernel/kernel/Scheduler.cpp index 445072bf90..a0db820e33 100644 --- a/kernel/kernel/Scheduler.cpp +++ b/kernel/kernel/Scheduler.cpp @@ -294,7 +294,7 @@ namespace Kernel Processor::yield(); } - void Scheduler::preempt() + void Scheduler::timer_interrupt() { ASSERT(Processor::get_interrupt_state() == InterruptState::Disabled); @@ -321,23 +321,6 @@ namespace Kernel } } - void Scheduler::timer_interrupt() - { - ASSERT(Processor::get_interrupt_state() == InterruptState::Disabled); - - // FIXME: all processors should LAPIC for their preemption - if (Processor::is_smp_enabled()) - { - ASSERT(Processor::current_is_bsb()); - Processor::broadcast_smp_message({ - .type = Processor::SMPMessage::Type::SchedulerPreemption, - .scheduler_preemption = 0 // dummy value - }); - } - - preempt(); - } - void Scheduler::handle_unblock_request(const UnblockRequest& request) { ASSERT(Processor::get_interrupt_state() == InterruptState::Disabled); diff --git a/kernel/kernel/Timer/HPET.cpp b/kernel/kernel/Timer/HPET.cpp index c5b8e05dd8..4935163b03 100644 --- a/kernel/kernel/Timer/HPET.cpp +++ b/kernel/kernel/Timer/HPET.cpp @@ -254,9 +254,9 @@ namespace Kernel void HPET::handle_irq() { - auto& regs = registers(); - { + auto& regs = registers(); + SpinLockGuard _(m_lock); uint64_t current_ticks; @@ -272,7 +272,8 @@ namespace Kernel m_last_ticks = current_ticks; } - Processor::scheduler().timer_interrupt(); + if (should_invoke_scheduler()) + Processor::scheduler().timer_interrupt(); } uint64_t HPET::ms_since_boot() const @@ -303,4 +304,33 @@ namespace Kernel }; } + void HPET::pre_scheduler_sleep_ns(uint64_t ns) + { + auto& regs = registers(); + + const uint64_t target_ticks = BAN::Math::div_round_up(ns * FS_PER_NS, regs.counter_clk_period); + + if (m_is_64bit) + { + const uint64_t target_counter = regs.main_counter.full + target_ticks; + while (regs.main_counter.full < target_counter) + __builtin_ia32_pause(); + } + else + { + uint64_t elapsed_ticks = 0; + uint64_t last_counter = regs.main_counter.low; + while (elapsed_ticks < target_ticks) + { + const uint64_t current_counter = regs.main_counter.low; + if (last_counter <= current_counter) + elapsed_ticks += current_counter - last_counter; + else + elapsed_ticks += 0xFFFFFFFF + current_counter - last_counter; + last_counter = current_counter; + __builtin_ia32_pause(); + } + } + } + } diff --git a/kernel/kernel/Timer/PIT.cpp b/kernel/kernel/Timer/PIT.cpp index ec15b5f102..c4009fc66b 100644 --- a/kernel/kernel/Timer/PIT.cpp +++ b/kernel/kernel/Timer/PIT.cpp @@ -18,17 +18,19 @@ #define ACCESS_HI 0x10 #define ACCESS_LO 0x20 -#define MODE_SQUARE_WAVE 0x06 +#define MODE_RATE_GENERATOR 0x05 #define BASE_FREQUENCY 1193182 -#define TICKS_PER_SECOND 1000 #define MS_PER_S 1'000 +#define NS_PER_MS 1'000'000 #define NS_PER_S 1'000'000'000 namespace Kernel { + constexpr uint16_t s_ticks_per_ms = BASE_FREQUENCY / 1000; + BAN::ErrorOr> PIT::create() { PIT* pit = new PIT(); @@ -40,12 +42,9 @@ namespace Kernel void PIT::initialize() { - constexpr uint16_t timer_reload = BASE_FREQUENCY / TICKS_PER_SECOND; - - IO::outb(PIT_CTL, SELECT_CHANNEL0 | ACCESS_LO | ACCESS_HI | MODE_SQUARE_WAVE); - - IO::outb(TIMER0_CTL, (timer_reload >> 0) & 0xff); - IO::outb(TIMER0_CTL, (timer_reload >> 8) & 0xff); + IO::outb(PIT_CTL, SELECT_CHANNEL0 | ACCESS_LO | ACCESS_HI | MODE_RATE_GENERATOR); + IO::outb(TIMER0_CTL, (s_ticks_per_ms >> 0) & 0xff); + IO::outb(TIMER0_CTL, (s_ticks_per_ms >> 8) & 0xff); MUST(InterruptController::get().reserve_irq(PIT_IRQ)); set_irq(PIT_IRQ); @@ -56,34 +55,77 @@ namespace Kernel { { SpinLockGuard _(m_lock); - m_system_time++; + m_system_time_ms++; } - Processor::scheduler().timer_interrupt(); + + if (should_invoke_scheduler()) + Processor::scheduler().timer_interrupt(); } - uint64_t PIT::read_counter() const + uint64_t PIT::read_counter_ns() const { SpinLockGuard _(m_lock); - return m_system_time; + + // send latch command + IO::outb(PIT_CTL, SELECT_CHANNEL0); + + // read ticks + uint64_t ticks_this_ms { 0 }; + ticks_this_ms |= IO::inb(TIMER0_CTL); + ticks_this_ms |= IO::inb(TIMER0_CTL) << 8; + ticks_this_ms = s_ticks_per_ms - ticks_this_ms; + + const uint64_t ns_this_ms = ticks_this_ms * NS_PER_S / BASE_FREQUENCY; + return (m_system_time_ms * NS_PER_MS) + ns_this_ms; } uint64_t PIT::ms_since_boot() const { - return read_counter() * (MS_PER_S / TICKS_PER_SECOND); + return read_counter_ns() / NS_PER_MS; } uint64_t PIT::ns_since_boot() const { - return read_counter() * (NS_PER_S / TICKS_PER_SECOND); + return read_counter_ns(); } timespec PIT::time_since_boot() const { - uint64_t ticks = read_counter(); + uint64_t ns = read_counter_ns(); return timespec { - .tv_sec = ticks / TICKS_PER_SECOND, - .tv_nsec = (long)((ticks % TICKS_PER_SECOND) * (NS_PER_S / TICKS_PER_SECOND)) + .tv_sec = static_cast(ns / NS_PER_S), + .tv_nsec = static_cast(ns % NS_PER_S) }; } + void PIT::pre_scheduler_sleep_ns(uint64_t ns) + { + const uint64_t target_ticks = BAN::Math::div_round_up(ns * BASE_FREQUENCY, NS_PER_S); + + SpinLockGuard _(m_lock); + + IO::outb(PIT_CTL, SELECT_CHANNEL0 | ACCESS_LO | ACCESS_HI | MODE_RATE_GENERATOR); + IO::outb(TIMER0_CTL, 0); + IO::outb(TIMER0_CTL, 0); + + IO::outb(PIT_CTL, SELECT_CHANNEL0 | ACCESS_LO | MODE_RATE_GENERATOR); + IO::outb(TIMER0_CTL, 0xFF); + + uint64_t elapsed_ticks = 0; + uint8_t last_ticks = IO::inb(TIMER0_CTL); + while (elapsed_ticks < target_ticks) + { + const uint8_t current_ticks = IO::inb(TIMER0_CTL); + if (last_ticks <= current_ticks) + elapsed_ticks += current_ticks - last_ticks; + else + elapsed_ticks += 0xFF + current_ticks - last_ticks; + last_ticks = current_ticks; + } + + IO::outb(PIT_CTL, SELECT_CHANNEL0 | ACCESS_LO | ACCESS_HI | MODE_RATE_GENERATOR); + IO::outb(TIMER0_CTL, (s_ticks_per_ms >> 0) & 0xFF); + IO::outb(TIMER0_CTL, (s_ticks_per_ms >> 8) & 0xFF); + } + } diff --git a/kernel/kernel/Timer/Timer.cpp b/kernel/kernel/Timer/Timer.cpp index 1c3186f6f1..d4075bdfc9 100644 --- a/kernel/kernel/Timer/Timer.cpp +++ b/kernel/kernel/Timer/Timer.cpp @@ -69,6 +69,11 @@ namespace Kernel return m_timer->time_since_boot(); } + void SystemTimer::pre_scheduler_sleep_ns(uint64_t ns) + { + return m_timer->pre_scheduler_sleep_ns(ns); + } + void SystemTimer::sleep_ns(uint64_t ns) const { if (ns == 0) @@ -76,10 +81,6 @@ namespace Kernel const uint64_t wake_time_ns = ns_since_boot() + ns; Processor::scheduler().block_current_thread(nullptr, wake_time_ns); - - //const uint64_t current_time_ns = ns_since_boot(); - //if (current_time_ns < wake_time_ns) - // dwarnln("sleep woke {} ms too soon", BAN::Math::div_round_up(wake_time_ns - current_time_ns, 1'000'000)); } timespec SystemTimer::real_time() const diff --git a/kernel/kernel/kernel.cpp b/kernel/kernel/kernel.cpp index 753c32b899..3e84b9f164 100644 --- a/kernel/kernel/kernel.cpp +++ b/kernel/kernel/kernel.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -167,6 +168,12 @@ extern "C" void kernel_main(uint32_t boot_magic, uint32_t boot_info) Random::initialize(); dprintln("RNG initialized"); + if (InterruptController::get().is_using_apic()) + { + SystemTimer::get().dont_invoke_scheduler(); + static_cast(InterruptController::get()).initialize_timer(); + } + Processor::wait_until_processors_ready(); MUST(Processor::scheduler().initialize());