Kernel: All processors use LAPIC timer when running with APIC

This makes scheduler preemption much cleaner as bsb does not have to send smp messages to notify other processes about timer interrupt. Also PIT percision is now "full" 0.8 us instead of 1 ms that I was using before.
2024-07-23 02:28:52 +03:00
parent 3e0150f847
commit 539afb329a
17 changed files with 224 additions and 73 deletions
--- a/kernel/kernel/APIC.cpp
+++ b/kernel/kernel/APIC.cpp
@@ -17,6 +17,11 @@
 #define LAPIC_ICR_LO_REG	0x300
 #define LAPIC_ICR_HI_REG	0x310

+#define LAPIC_TIMER_LVT			0x320
+#define LAPIC_TIMER_INITIAL_REG	0x380
+#define LAPIC_TIMER_CURRENT_REG	0x390
+#define LAPIC_TIMER_DIVIDE_REG	0x3E0
+
 #define IOAPIC_MAX_REDIRS	0x01
 #define IOAPIC_REDIRS		0x10

@@ -60,6 +65,27 @@ namespace Kernel
 		ICR_LO_destination_shorthand_all_excluding_self	= 0b11 << 18,
 	};

+	enum TimerLVT : uint32_t
+	{
+		TimerModeOneShot     = 0b00 << 17,
+		TimerModePeriodic    = 0b01 << 17,
+		TimerModeTSCDeadline = 0b10 << 17,
+
+		TimerMask            = 1 << 16,
+	};
+
+	enum TimerDivideRegister : uint32_t
+	{
+		DivideBy2   = 0b0000,
+		DivideBy4   = 0b0001,
+		DivideBy8   = 0b0010,
+		DivideBy16  = 0b0011,
+		DivideBy32  = 0b1000,
+		DivideBy64  = 0b1001,
+		DivideBy128 = 0b1010,
+		DivideBy1   = 0b1011,
+	};
+
 	struct MADT : public Kernel::ACPI::SDTHeader
 	{
 		uint32_t local_apic;
@@ -223,12 +249,7 @@ namespace Kernel

 	void APIC::initialize_multiprocessor()
 	{
-		constexpr auto udelay =
-			[](uint64_t us) {
-				uint64_t wake_time = SystemTimer::get().ns_since_boot() + us * 1000;
-				while (SystemTimer::get().ns_since_boot() < wake_time)
-					__builtin_ia32_pause();
-			};
+		constexpr auto udelay = [](uint64_t us) { SystemTimer::get().pre_scheduler_sleep_ns(us * 1000); };

 		const auto send_ipi =
 			[&](uint8_t processor, uint32_t data, uint64_t ud)
@@ -367,6 +388,31 @@ namespace Kernel
 	void APIC::enable()
 	{
 		write_to_local_apic(LAPIC_SIV_REG, read_from_local_apic(LAPIC_SIV_REG) | 0x1FF);
+		initialize_timer();
+	}
+
+	static SpinLock s_timer_init_lock;
+
+	void APIC::initialize_timer()
+	{
+		{
+			constexpr uint64_t measuring_duration_ms = 100;
+
+			SpinLockGuard _(s_timer_init_lock);
+
+			write_to_local_apic(LAPIC_TIMER_LVT,         TimerModeOneShot | TimerMask);
+			write_to_local_apic(LAPIC_TIMER_DIVIDE_REG,  DivideBy2);
+			write_to_local_apic(LAPIC_TIMER_INITIAL_REG, 0xFFFFFFFF);
+			SystemTimer::get().pre_scheduler_sleep_ns(measuring_duration_ms * 1'000'000);
+
+			const uint32_t counter = read_from_local_apic(LAPIC_TIMER_CURRENT_REG);
+			m_lapic_timer_frequency_hz = static_cast<uint64_t>(0xFFFFFFFF - counter) * 2 * (1000 / measuring_duration_ms);
+
+			dprintln("CPU {}: lapic timer frequency: {} Hz", Kernel::Processor::current_id(), m_lapic_timer_frequency_hz);
+		}
+
+		write_to_local_apic(LAPIC_TIMER_LVT,         TimerModePeriodic | (IRQ_VECTOR_BASE + IRQ_TIMER));
+		write_to_local_apic(LAPIC_TIMER_INITIAL_REG, m_lapic_timer_frequency_hz / 2 / 100);
 	}

 	uint32_t APIC::read_from_local_apic(ptrdiff_t offset)
--- a/kernel/kernel/IDT.cpp
+++ b/kernel/kernel/IDT.cpp
@@ -333,6 +333,13 @@ done:
 		Processor::handle_ipi();
 	}

+	extern "C" void cpp_timer_handler()
+	{
+		ASSERT(InterruptController::get().is_in_service(IRQ_TIMER));
+		InterruptController::get().eoi(IRQ_TIMER);
+		Processor::scheduler().timer_interrupt();
+	}
+
 	extern "C" void cpp_irq_handler(uint32_t irq)
 	{
 		if (g_paniced)
@@ -409,6 +416,7 @@ done:

 	extern "C" void asm_yield_handler();
 	extern "C" void asm_ipi_handler();
+	extern "C" void asm_timer_handler();
 	extern "C" void asm_syscall_handler();

 	IDT* IDT::create()
@@ -428,6 +436,7 @@ done:

 		idt->register_interrupt_handler(IRQ_VECTOR_BASE + IRQ_YIELD, asm_yield_handler);
 		idt->register_interrupt_handler(IRQ_VECTOR_BASE + IRQ_IPI,   asm_ipi_handler);
+		idt->register_interrupt_handler(IRQ_VECTOR_BASE + IRQ_TIMER, asm_timer_handler);

 		idt->register_syscall_handler(0x80, asm_syscall_handler);

--- a/kernel/kernel/Processor.cpp
+++ b/kernel/kernel/Processor.cpp
@@ -20,7 +20,7 @@ namespace Kernel
 	static BAN::Atomic<uint8_t>  s_processors_created { 0 };

 	// 32 bit milli seconds are definitely enough as APs start on boot
-	static BAN::Atomic<uint32_t> s_first_ap_ready_ms { 0 };
+	static BAN::Atomic<uint32_t> s_first_ap_ready_ms { static_cast<uint32_t>(-1) };

 	static BAN::Array<Processor,   0xFF> s_processors;
 	static BAN::Array<ProcessorID, 0xFF> s_processor_ids { PROCESSOR_NONE };
@@ -130,7 +130,7 @@ namespace Kernel

 			// wait until first AP is ready
 			const uint64_t timeout_ms = SystemTimer::get().ms_since_boot() + 1000;
-			while (s_first_ap_ready_ms == 0)
+			while (s_first_ap_ready_ms == static_cast<uint32_t>(-1))
 			{
 				if (SystemTimer::get().ms_since_boot() >= timeout_ms)
 				{
@@ -150,7 +150,7 @@ namespace Kernel
 			ASSERT(s_processor_ids[lookup_index] == PROCESSOR_NONE);
 			s_processor_ids[lookup_index] = current_id();

-			uint32_t expected = 0;
+			uint32_t expected = static_cast<uint32_t>(-1);
 			s_first_ap_ready_ms.compare_exchange(expected, SystemTimer::get().ms_since_boot());
 		}

@@ -209,8 +209,6 @@ namespace Kernel
 			}
 		);

-		bool should_preempt = false;
-
 		if (pending)
 		{
 			// reverse smp message queue from LIFO to FIFO
@@ -245,9 +243,6 @@ namespace Kernel
 					case SMPMessage::Type::UnblockThread:
 						processor.m_scheduler->handle_unblock_request(message->unblock_thread);
 						break;
-					case SMPMessage::Type::SchedulerPreemption:
-						should_preempt = true;
-						break;
 				}

 				last_handled = message;
@@ -262,9 +257,6 @@ namespace Kernel
 			);
 		}

-		if (should_preempt)
-			processor.m_scheduler->preempt();
-
 		set_interrupt_state(state);
 	}

--- a/kernel/kernel/Scheduler.cpp
+++ b/kernel/kernel/Scheduler.cpp
@@ -294,7 +294,7 @@ namespace Kernel
 			Processor::yield();
 	}

-	void Scheduler::preempt()
+	void Scheduler::timer_interrupt()
 	{
 		ASSERT(Processor::get_interrupt_state() == InterruptState::Disabled);

@@ -321,23 +321,6 @@ namespace Kernel
 		}
 	}

-	void Scheduler::timer_interrupt()
-	{
-		ASSERT(Processor::get_interrupt_state() == InterruptState::Disabled);
-
-		// FIXME: all processors should LAPIC for their preemption
-		if (Processor::is_smp_enabled())
-		{
-			ASSERT(Processor::current_is_bsb());
-			Processor::broadcast_smp_message({
-				.type = Processor::SMPMessage::Type::SchedulerPreemption,
-				.scheduler_preemption = 0 // dummy value
-			});
-		}
-
-		preempt();
-	}
-
 	void Scheduler::handle_unblock_request(const UnblockRequest& request)
 	{
 		ASSERT(Processor::get_interrupt_state() == InterruptState::Disabled);
--- a/kernel/kernel/Timer/HPET.cpp
+++ b/kernel/kernel/Timer/HPET.cpp
@@ -254,9 +254,9 @@ namespace Kernel

 	void HPET::handle_irq()
 	{
-		auto& regs = registers();
-
 		{
+			auto& regs = registers();
+
 			SpinLockGuard _(m_lock);

 			uint64_t current_ticks;
@@ -272,7 +272,8 @@ namespace Kernel
 			m_last_ticks = current_ticks;
 		}

-		Processor::scheduler().timer_interrupt();
+		if (should_invoke_scheduler())
+			Processor::scheduler().timer_interrupt();
 	}

 	uint64_t HPET::ms_since_boot() const
@@ -303,4 +304,33 @@ namespace Kernel
 		};
 	}

+	void HPET::pre_scheduler_sleep_ns(uint64_t ns)
+	{
+		auto& regs = registers();
+
+		const uint64_t target_ticks = BAN::Math::div_round_up<uint64_t>(ns * FS_PER_NS, regs.counter_clk_period);
+
+		if (m_is_64bit)
+		{
+			const uint64_t target_counter = regs.main_counter.full + target_ticks;
+			while (regs.main_counter.full < target_counter)
+				__builtin_ia32_pause();
+		}
+		else
+		{
+			uint64_t elapsed_ticks = 0;
+			uint64_t last_counter = regs.main_counter.low;
+			while (elapsed_ticks < target_ticks)
+			{
+				const uint64_t current_counter = regs.main_counter.low;
+				if (last_counter <= current_counter)
+					elapsed_ticks += current_counter - last_counter;
+				else
+					elapsed_ticks += 0xFFFFFFFF + current_counter - last_counter;
+				last_counter = current_counter;
+				__builtin_ia32_pause();
+			}
+		}
+	}
+
 }
--- a/kernel/kernel/Timer/PIT.cpp
+++ b/kernel/kernel/Timer/PIT.cpp
@@ -18,17 +18,19 @@
 #define ACCESS_HI 			0x10
 #define ACCESS_LO 			0x20

-#define MODE_SQUARE_WAVE	0x06
+#define MODE_RATE_GENERATOR	0x05

 #define BASE_FREQUENCY		1193182
-#define TICKS_PER_SECOND	1000

 #define MS_PER_S			1'000
+#define NS_PER_MS			1'000'000
 #define NS_PER_S			1'000'000'000

 namespace Kernel
 {

+	constexpr uint16_t s_ticks_per_ms = BASE_FREQUENCY / 1000;
+
 	BAN::ErrorOr<BAN::UniqPtr<PIT>> PIT::create()
 	{
 		PIT* pit = new PIT();
@@ -40,12 +42,9 @@ namespace Kernel

 	void PIT::initialize()
 	{
-		constexpr uint16_t timer_reload = BASE_FREQUENCY / TICKS_PER_SECOND;
-
-		IO::outb(PIT_CTL, SELECT_CHANNEL0 | ACCESS_LO | ACCESS_HI | MODE_SQUARE_WAVE);
-
-		IO::outb(TIMER0_CTL, (timer_reload >> 0) & 0xff);
-		IO::outb(TIMER0_CTL, (timer_reload >> 8) & 0xff);
+		IO::outb(PIT_CTL, SELECT_CHANNEL0 | ACCESS_LO | ACCESS_HI | MODE_RATE_GENERATOR);
+		IO::outb(TIMER0_CTL, (s_ticks_per_ms >> 0) & 0xff);
+		IO::outb(TIMER0_CTL, (s_ticks_per_ms >> 8) & 0xff);

 		MUST(InterruptController::get().reserve_irq(PIT_IRQ));
 		set_irq(PIT_IRQ);
@@ -56,34 +55,77 @@ namespace Kernel
 	{
 		{
 			SpinLockGuard _(m_lock);
-			m_system_time++;
+			m_system_time_ms++;
 		}
-		Processor::scheduler().timer_interrupt();
+
+		if (should_invoke_scheduler())
+			Processor::scheduler().timer_interrupt();
 	}

-	uint64_t PIT::read_counter() const
+	uint64_t PIT::read_counter_ns() const
 	{
 		SpinLockGuard _(m_lock);
-		return m_system_time;
+
+		// send latch command
+		IO::outb(PIT_CTL, SELECT_CHANNEL0);
+
+		// read ticks
+		uint64_t ticks_this_ms { 0 };
+		ticks_this_ms |= IO::inb(TIMER0_CTL);
+		ticks_this_ms |= IO::inb(TIMER0_CTL) << 8;
+		ticks_this_ms = s_ticks_per_ms - ticks_this_ms;
+
+		const uint64_t ns_this_ms = ticks_this_ms * NS_PER_S / BASE_FREQUENCY;
+		return (m_system_time_ms * NS_PER_MS) + ns_this_ms;
 	}

 	uint64_t PIT::ms_since_boot() const
 	{
-		return read_counter() * (MS_PER_S / TICKS_PER_SECOND);
+		return read_counter_ns() / NS_PER_MS;
 	}

 	uint64_t PIT::ns_since_boot() const
 	{
-		return read_counter() * (NS_PER_S / TICKS_PER_SECOND);
+		return read_counter_ns();
 	}

 	timespec PIT::time_since_boot() const
 	{
-		uint64_t ticks = read_counter();
+		uint64_t ns = read_counter_ns();
 		return timespec {
-			.tv_sec = ticks / TICKS_PER_SECOND,
-			.tv_nsec = (long)((ticks % TICKS_PER_SECOND) * (NS_PER_S / TICKS_PER_SECOND))
+			.tv_sec = static_cast<time_t>(ns / NS_PER_S),
+			.tv_nsec = static_cast<long>(ns % NS_PER_S)
 		};
 	}

+	void PIT::pre_scheduler_sleep_ns(uint64_t ns)
+	{
+		const uint64_t target_ticks = BAN::Math::div_round_up<uint64_t>(ns * BASE_FREQUENCY, NS_PER_S);
+
+		SpinLockGuard _(m_lock);
+
+		IO::outb(PIT_CTL, SELECT_CHANNEL0 | ACCESS_LO | ACCESS_HI | MODE_RATE_GENERATOR);
+		IO::outb(TIMER0_CTL, 0);
+		IO::outb(TIMER0_CTL, 0);
+
+		IO::outb(PIT_CTL, SELECT_CHANNEL0 | ACCESS_LO | MODE_RATE_GENERATOR);
+		IO::outb(TIMER0_CTL, 0xFF);
+
+		uint64_t elapsed_ticks = 0;
+		uint8_t last_ticks = IO::inb(TIMER0_CTL);
+		while (elapsed_ticks < target_ticks)
+		{
+			const uint8_t current_ticks = IO::inb(TIMER0_CTL);
+			if (last_ticks <= current_ticks)
+				elapsed_ticks += current_ticks - last_ticks;
+			else
+				elapsed_ticks += 0xFF + current_ticks - last_ticks;
+			last_ticks = current_ticks;
+		}
+
+		IO::outb(PIT_CTL, SELECT_CHANNEL0 | ACCESS_LO | ACCESS_HI | MODE_RATE_GENERATOR);
+		IO::outb(TIMER0_CTL, (s_ticks_per_ms >> 0) & 0xFF);
+		IO::outb(TIMER0_CTL, (s_ticks_per_ms >> 8) & 0xFF);
+	}
+
 }
--- a/kernel/kernel/Timer/Timer.cpp
+++ b/kernel/kernel/Timer/Timer.cpp
@@ -69,6 +69,11 @@ namespace Kernel
 		return m_timer->time_since_boot();
 	}

+	void SystemTimer::pre_scheduler_sleep_ns(uint64_t ns)
+	{
+		return m_timer->pre_scheduler_sleep_ns(ns);
+	}
+
 	void SystemTimer::sleep_ns(uint64_t ns) const
 	{
 		if (ns == 0)
@@ -76,10 +81,6 @@ namespace Kernel

 		const uint64_t wake_time_ns = ns_since_boot() + ns;
 		Processor::scheduler().block_current_thread(nullptr, wake_time_ns);
-
-		//const uint64_t current_time_ns = ns_since_boot();
-		//if (current_time_ns < wake_time_ns)
-		//	dwarnln("sleep woke {} ms too soon", BAN::Math::div_round_up<uint64_t>(wake_time_ns - current_time_ns, 1'000'000));
 	}

 	timespec SystemTimer::real_time() const
--- a/kernel/kernel/kernel.cpp
+++ b/kernel/kernel/kernel.cpp
@@ -1,4 +1,5 @@
 #include <kernel/ACPI/ACPI.h>
+#include <kernel/APIC.h>
 #include <kernel/Arch.h>
 #include <kernel/BootInfo.h>
 #include <kernel/Debug.h>
@@ -167,6 +168,12 @@ extern "C" void kernel_main(uint32_t boot_magic, uint32_t boot_info)
 	Random::initialize();
 	dprintln("RNG initialized");

+	if (InterruptController::get().is_using_apic())
+	{
+		SystemTimer::get().dont_invoke_scheduler();
+		static_cast<APIC&>(InterruptController::get()).initialize_timer();
+	}
+
 	Processor::wait_until_processors_ready();
 	MUST(Processor::scheduler().initialize());