Kernel: All processors use LAPIC timer when running with APIC

This makes scheduler preemption much cleaner as bsb does not have to
send smp messages to notify other processes about timer interrupt.

Also PIT percision is now "full" 0.8 us instead of 1 ms that I was using
before.

kernel/arch/i686/interrupts.S

@@ -113,6 +113,23 @@ asm_ipi_handler:
 	pop_userspace
 	iret
 
+
+.global asm_timer_handler
+asm_timer_handler:
+	push_userspace
+	load_kernel_segments
+
+	movl %esp, %ebp
+	subl $15, %esp
+	andl $0xFFFFFFF0, %esp
+
+	call cpp_timer_handler
+
+	movl %ebp, %esp
+
+	pop_userspace
+	iret
+
 .macro isr n
 	.global isr\n
 	isr\n:

kernel/arch/x86_64/interrupts.S

@@ -77,6 +77,13 @@ asm_ipi_handler:
 	popaq
 	iretq
 
+.global asm_timer_handler
+asm_timer_handler:
+	pushaq
+	call cpp_timer_handler
+	popaq
+	iretq
+
 .macro isr n
 	.global isr\n
 	isr\n:

kernel/include/kernel/APIC.h

@@ -23,6 +23,8 @@ namespace Kernel
 		virtual void broadcast_ipi() override;
 		virtual void enable() override;
 
+		void initialize_timer();
+
 	private:
 		uint32_t read_from_local_apic(ptrdiff_t);
 		void write_to_local_apic(ptrdiff_t, uint32_t);
@@ -65,6 +67,7 @@ namespace Kernel
 		BAN::Vector<IOAPIC>    m_io_apics;
 		uint8_t                m_irq_overrides[0x100] {};
 		uint8_t                m_reserved_gsis[0x100 / 8] {};
+		uint64_t               m_lapic_timer_frequency_hz { 0 };
 	};
 
 }

kernel/include/kernel/IDT.h

@@ -10,6 +10,7 @@
 constexpr uint8_t IRQ_VECTOR_BASE   = 0x20;
 constexpr uint8_t IRQ_YIELD         = 32;
 constexpr uint8_t IRQ_IPI           = 33;
+constexpr uint8_t IRQ_TIMER         = 34;
 
 namespace Kernel
 {

kernel/include/kernel/Processor.h

@@ -56,8 +56,6 @@ namespace Kernel
 				FlushTLB,
 				NewThread,
 				UnblockThread,
-				// FIXME: all processors should LAPIC for their preemption
-				SchedulerPreemption,
 			};
 			SMPMessage* next { nullptr };
 			Type type;

kernel/include/kernel/Scheduler.h

@@ -110,8 +110,6 @@ namespace Kernel
 
 		class ProcessorID find_least_loaded_processor() const;
 
-		void preempt();
-
 		void handle_unblock_request(const UnblockRequest&);
 		void handle_new_thread_request(const NewThreadRequest&);
 

kernel/include/kernel/Timer/HPET.h

@@ -19,6 +19,8 @@ namespace Kernel
 		virtual uint64_t ns_since_boot() const override;
 		virtual timespec time_since_boot() const override;
 
+		virtual void pre_scheduler_sleep_ns(uint64_t) override;
+
 		virtual void handle_irq() override;
 
 	private:

kernel/include/kernel/Timer/PIT.h

@@ -16,15 +16,18 @@ namespace Kernel
 		virtual uint64_t ns_since_boot() const override;
 		virtual timespec time_since_boot() const override;
 
+		virtual void pre_scheduler_sleep_ns(uint64_t) override;
+
 		virtual void handle_irq() override;
 
 	private:
 		void initialize();
-		uint64_t read_counter() const;
+
+		uint64_t read_counter_ns() const;
 
 	private:
 		mutable SpinLock m_lock;
-		uint64_t m_system_time { 0 };
+		uint64_t m_system_time_ms { 0 };
 	};
 
 }

kernel/include/kernel/Timer/Timer.h

@@ -16,6 +16,16 @@ namespace Kernel
 		virtual uint64_t ms_since_boot() const = 0;
 		virtual uint64_t ns_since_boot() const = 0;
 		virtual timespec time_since_boot() const = 0;
+
+		virtual void pre_scheduler_sleep_ns(uint64_t) = 0;
+
+		void dont_invoke_scheduler() { m_should_invoke_scheduler = false; }
+
+	protected:
+		bool should_invoke_scheduler() const { return m_should_invoke_scheduler; }
+
+	private:
+		bool m_should_invoke_scheduler { true };
 	};
 
 	class SystemTimer : public Timer
@@ -29,6 +39,8 @@ namespace Kernel
 		virtual uint64_t ns_since_boot() const override;
 		virtual timespec time_since_boot() const override;
 
+		virtual void pre_scheduler_sleep_ns(uint64_t) override;
+
 		void sleep_ms(uint64_t ms) const { return sleep_ns(ms * 1'000'000); }
 		void sleep_ns(uint64_t ns) const;
 

kernel/kernel/APIC.cpp

@@ -17,6 +17,11 @@
 #define LAPIC_ICR_LO_REG	0x300
 #define LAPIC_ICR_HI_REG	0x310
 
+#define LAPIC_TIMER_LVT			0x320
+#define LAPIC_TIMER_INITIAL_REG	0x380
+#define LAPIC_TIMER_CURRENT_REG	0x390
+#define LAPIC_TIMER_DIVIDE_REG	0x3E0
+
 #define IOAPIC_MAX_REDIRS	0x01
 #define IOAPIC_REDIRS		0x10
 
@@ -60,6 +65,27 @@ namespace Kernel
 		ICR_LO_destination_shorthand_all_excluding_self	= 0b11 << 18,
 	};
 
+	enum TimerLVT : uint32_t
+	{
+		TimerModeOneShot     = 0b00 << 17,
+		TimerModePeriodic    = 0b01 << 17,
+		TimerModeTSCDeadline = 0b10 << 17,
+
+		TimerMask            = 1 << 16,
+	};
+
+	enum TimerDivideRegister : uint32_t
+	{
+		DivideBy2   = 0b0000,
+		DivideBy4   = 0b0001,
+		DivideBy8   = 0b0010,
+		DivideBy16  = 0b0011,
+		DivideBy32  = 0b1000,
+		DivideBy64  = 0b1001,
+		DivideBy128 = 0b1010,
+		DivideBy1   = 0b1011,
+	};
+
 	struct MADT : public Kernel::ACPI::SDTHeader
 	{
 		uint32_t local_apic;
@@ -223,12 +249,7 @@ namespace Kernel
 
 	void APIC::initialize_multiprocessor()
 	{
-		constexpr auto udelay =
-			[](uint64_t us) {
-				uint64_t wake_time = SystemTimer::get().ns_since_boot() + us * 1000;
-				while (SystemTimer::get().ns_since_boot() < wake_time)
-					__builtin_ia32_pause();
-			};
+		constexpr auto udelay = [](uint64_t us) { SystemTimer::get().pre_scheduler_sleep_ns(us * 1000); };
 
 		const auto send_ipi =
 			[&](uint8_t processor, uint32_t data, uint64_t ud)
@@ -367,6 +388,31 @@ namespace Kernel
 	void APIC::enable()
 	{
 		write_to_local_apic(LAPIC_SIV_REG, read_from_local_apic(LAPIC_SIV_REG) | 0x1FF);
+		initialize_timer();
+	}
+
+	static SpinLock s_timer_init_lock;
+
+	void APIC::initialize_timer()
+	{
+		{
+			constexpr uint64_t measuring_duration_ms = 100;
+
+			SpinLockGuard _(s_timer_init_lock);
+
+			write_to_local_apic(LAPIC_TIMER_LVT,         TimerModeOneShot | TimerMask);
+			write_to_local_apic(LAPIC_TIMER_DIVIDE_REG,  DivideBy2);
+			write_to_local_apic(LAPIC_TIMER_INITIAL_REG, 0xFFFFFFFF);
+			SystemTimer::get().pre_scheduler_sleep_ns(measuring_duration_ms * 1'000'000);
+
+			const uint32_t counter = read_from_local_apic(LAPIC_TIMER_CURRENT_REG);
+			m_lapic_timer_frequency_hz = static_cast<uint64_t>(0xFFFFFFFF - counter) * 2 * (1000 / measuring_duration_ms);
+
+			dprintln("CPU {}: lapic timer frequency: {} Hz", Kernel::Processor::current_id(), m_lapic_timer_frequency_hz);
+		}
+
+		write_to_local_apic(LAPIC_TIMER_LVT,         TimerModePeriodic | (IRQ_VECTOR_BASE + IRQ_TIMER));
+		write_to_local_apic(LAPIC_TIMER_INITIAL_REG, m_lapic_timer_frequency_hz / 2 / 100);
 	}
 
 	uint32_t APIC::read_from_local_apic(ptrdiff_t offset)

kernel/kernel/IDT.cpp

@@ -333,6 +333,13 @@ done:
 		Processor::handle_ipi();
 	}
 
+	extern "C" void cpp_timer_handler()
+	{
+		ASSERT(InterruptController::get().is_in_service(IRQ_TIMER));
+		InterruptController::get().eoi(IRQ_TIMER);
+		Processor::scheduler().timer_interrupt();
+	}
+
 	extern "C" void cpp_irq_handler(uint32_t irq)
 	{
 		if (g_paniced)
@@ -409,6 +416,7 @@ done:
 
 	extern "C" void asm_yield_handler();
 	extern "C" void asm_ipi_handler();
+	extern "C" void asm_timer_handler();
 	extern "C" void asm_syscall_handler();
 
 	IDT* IDT::create()
@@ -428,6 +436,7 @@ done:
 
 		idt->register_interrupt_handler(IRQ_VECTOR_BASE + IRQ_YIELD, asm_yield_handler);
 		idt->register_interrupt_handler(IRQ_VECTOR_BASE + IRQ_IPI,   asm_ipi_handler);
+		idt->register_interrupt_handler(IRQ_VECTOR_BASE + IRQ_TIMER, asm_timer_handler);
 
 		idt->register_syscall_handler(0x80, asm_syscall_handler);
 

kernel/kernel/Processor.cpp

@@ -20,7 +20,7 @@ namespace Kernel
 	static BAN::Atomic<uint8_t>  s_processors_created { 0 };
 
 	// 32 bit milli seconds are definitely enough as APs start on boot
-	static BAN::Atomic<uint32_t> s_first_ap_ready_ms { 0 };
+	static BAN::Atomic<uint32_t> s_first_ap_ready_ms { static_cast<uint32_t>(-1) };
 
 	static BAN::Array<Processor,   0xFF> s_processors;
 	static BAN::Array<ProcessorID, 0xFF> s_processor_ids { PROCESSOR_NONE };
@@ -130,7 +130,7 @@ namespace Kernel
 
 			// wait until first AP is ready
 			const uint64_t timeout_ms = SystemTimer::get().ms_since_boot() + 1000;
-			while (s_first_ap_ready_ms == 0)
+			while (s_first_ap_ready_ms == static_cast<uint32_t>(-1))
 			{
 				if (SystemTimer::get().ms_since_boot() >= timeout_ms)
 				{
@@ -150,7 +150,7 @@ namespace Kernel
 			ASSERT(s_processor_ids[lookup_index] == PROCESSOR_NONE);
 			s_processor_ids[lookup_index] = current_id();
 
-			uint32_t expected = 0;
+			uint32_t expected = static_cast<uint32_t>(-1);
 			s_first_ap_ready_ms.compare_exchange(expected, SystemTimer::get().ms_since_boot());
 		}
 
@@ -209,8 +209,6 @@ namespace Kernel
 			}
 		);
 
-		bool should_preempt = false;
-
 		if (pending)
 		{
 			// reverse smp message queue from LIFO to FIFO
@@ -245,9 +243,6 @@ namespace Kernel
 					case SMPMessage::Type::UnblockThread:
 						processor.m_scheduler->handle_unblock_request(message->unblock_thread);
 						break;
-					case SMPMessage::Type::SchedulerPreemption:
-						should_preempt = true;
-						break;
 				}
 
 				last_handled = message;
@@ -262,9 +257,6 @@ namespace Kernel
 			);
 		}
 
-		if (should_preempt)
-			processor.m_scheduler->preempt();
-
 		set_interrupt_state(state);
 	}
 

kernel/kernel/Scheduler.cpp

@@ -294,7 +294,7 @@ namespace Kernel
 			Processor::yield();
 	}
 
-	void Scheduler::preempt()
+	void Scheduler::timer_interrupt()
 	{
 		ASSERT(Processor::get_interrupt_state() == InterruptState::Disabled);
 
@@ -321,23 +321,6 @@ namespace Kernel
 		}
 	}
 
-	void Scheduler::timer_interrupt()
-	{
-		ASSERT(Processor::get_interrupt_state() == InterruptState::Disabled);
-
-		// FIXME: all processors should LAPIC for their preemption
-		if (Processor::is_smp_enabled())
-		{
-			ASSERT(Processor::current_is_bsb());
-			Processor::broadcast_smp_message({
-				.type = Processor::SMPMessage::Type::SchedulerPreemption,
-				.scheduler_preemption = 0 // dummy value
-			});
-		}
-
-		preempt();
-	}
-
 	void Scheduler::handle_unblock_request(const UnblockRequest& request)
 	{
 		ASSERT(Processor::get_interrupt_state() == InterruptState::Disabled);

kernel/kernel/Timer/HPET.cpp

@@ -253,10 +253,10 @@ namespace Kernel
 	}
 
 	void HPET::handle_irq()
+	{
 		{
 			auto& regs = registers();
 
-		{
 			SpinLockGuard _(m_lock);
 
 			uint64_t current_ticks;
@@ -272,6 +272,7 @@ namespace Kernel
 			m_last_ticks = current_ticks;
 		}
 
+		if (should_invoke_scheduler())
 			Processor::scheduler().timer_interrupt();
 	}
 
@@ -303,4 +304,33 @@ namespace Kernel
 		};
 	}
 
+	void HPET::pre_scheduler_sleep_ns(uint64_t ns)
+	{
+		auto& regs = registers();
+
+		const uint64_t target_ticks = BAN::Math::div_round_up<uint64_t>(ns * FS_PER_NS, regs.counter_clk_period);
+
+		if (m_is_64bit)
+		{
+			const uint64_t target_counter = regs.main_counter.full + target_ticks;
+			while (regs.main_counter.full < target_counter)
+				__builtin_ia32_pause();
+		}
+		else
+		{
+			uint64_t elapsed_ticks = 0;
+			uint64_t last_counter = regs.main_counter.low;
+			while (elapsed_ticks < target_ticks)
+			{
+				const uint64_t current_counter = regs.main_counter.low;
+				if (last_counter <= current_counter)
+					elapsed_ticks += current_counter - last_counter;
+				else
+					elapsed_ticks += 0xFFFFFFFF + current_counter - last_counter;
+				last_counter = current_counter;
+				__builtin_ia32_pause();
+			}
+		}
+	}
+
 }

kernel/kernel/Timer/PIT.cpp

@@ -18,17 +18,19 @@
 #define ACCESS_HI 			0x10
 #define ACCESS_LO 			0x20
 
-#define MODE_SQUARE_WAVE	0x06
+#define MODE_RATE_GENERATOR	0x05
 
 #define BASE_FREQUENCY		1193182
-#define TICKS_PER_SECOND	1000
 
 #define MS_PER_S			1'000
+#define NS_PER_MS			1'000'000
 #define NS_PER_S			1'000'000'000
 
 namespace Kernel
 {
 
+	constexpr uint16_t s_ticks_per_ms = BASE_FREQUENCY / 1000;
+
 	BAN::ErrorOr<BAN::UniqPtr<PIT>> PIT::create()
 	{
 		PIT* pit = new PIT();
@@ -40,12 +42,9 @@ namespace Kernel
 
 	void PIT::initialize()
 	{
-		constexpr uint16_t timer_reload = BASE_FREQUENCY / TICKS_PER_SECOND;
-
-		IO::outb(PIT_CTL, SELECT_CHANNEL0 | ACCESS_LO | ACCESS_HI | MODE_SQUARE_WAVE);
-
-		IO::outb(TIMER0_CTL, (timer_reload >> 0) & 0xff);
-		IO::outb(TIMER0_CTL, (timer_reload >> 8) & 0xff);
+		IO::outb(PIT_CTL, SELECT_CHANNEL0 | ACCESS_LO | ACCESS_HI | MODE_RATE_GENERATOR);
+		IO::outb(TIMER0_CTL, (s_ticks_per_ms >> 0) & 0xff);
+		IO::outb(TIMER0_CTL, (s_ticks_per_ms >> 8) & 0xff);
 
 		MUST(InterruptController::get().reserve_irq(PIT_IRQ));
 		set_irq(PIT_IRQ);
@@ -56,34 +55,77 @@ namespace Kernel
 	{
 		{
 			SpinLockGuard _(m_lock);
-			m_system_time++;
+			m_system_time_ms++;
 		}
+
+		if (should_invoke_scheduler())
 			Processor::scheduler().timer_interrupt();
 	}
 
-	uint64_t PIT::read_counter() const
+	uint64_t PIT::read_counter_ns() const
 	{
 		SpinLockGuard _(m_lock);
-		return m_system_time;
+
+		// send latch command
+		IO::outb(PIT_CTL, SELECT_CHANNEL0);
+
+		// read ticks
+		uint64_t ticks_this_ms { 0 };
+		ticks_this_ms |= IO::inb(TIMER0_CTL);
+		ticks_this_ms |= IO::inb(TIMER0_CTL) << 8;
+		ticks_this_ms = s_ticks_per_ms - ticks_this_ms;
+
+		const uint64_t ns_this_ms = ticks_this_ms * NS_PER_S / BASE_FREQUENCY;
+		return (m_system_time_ms * NS_PER_MS) + ns_this_ms;
 	}
 
 	uint64_t PIT::ms_since_boot() const
 	{
-		return read_counter() * (MS_PER_S / TICKS_PER_SECOND);
+		return read_counter_ns() / NS_PER_MS;
 	}
 
 	uint64_t PIT::ns_since_boot() const
 	{
-		return read_counter() * (NS_PER_S / TICKS_PER_SECOND);
+		return read_counter_ns();
 	}
 
 	timespec PIT::time_since_boot() const
 	{
-		uint64_t ticks = read_counter();
+		uint64_t ns = read_counter_ns();
 		return timespec {
-			.tv_sec = ticks / TICKS_PER_SECOND,
-			.tv_nsec = (long)((ticks % TICKS_PER_SECOND) * (NS_PER_S / TICKS_PER_SECOND))
+			.tv_sec = static_cast<time_t>(ns / NS_PER_S),
+			.tv_nsec = static_cast<long>(ns % NS_PER_S)
 		};
 	}
 
+	void PIT::pre_scheduler_sleep_ns(uint64_t ns)
+	{
+		const uint64_t target_ticks = BAN::Math::div_round_up<uint64_t>(ns * BASE_FREQUENCY, NS_PER_S);
+
+		SpinLockGuard _(m_lock);
+
+		IO::outb(PIT_CTL, SELECT_CHANNEL0 | ACCESS_LO | ACCESS_HI | MODE_RATE_GENERATOR);
+		IO::outb(TIMER0_CTL, 0);
+		IO::outb(TIMER0_CTL, 0);
+
+		IO::outb(PIT_CTL, SELECT_CHANNEL0 | ACCESS_LO | MODE_RATE_GENERATOR);
+		IO::outb(TIMER0_CTL, 0xFF);
+
+		uint64_t elapsed_ticks = 0;
+		uint8_t last_ticks = IO::inb(TIMER0_CTL);
+		while (elapsed_ticks < target_ticks)
+		{
+			const uint8_t current_ticks = IO::inb(TIMER0_CTL);
+			if (last_ticks <= current_ticks)
+				elapsed_ticks += current_ticks - last_ticks;
+			else
+				elapsed_ticks += 0xFF + current_ticks - last_ticks;
+			last_ticks = current_ticks;
+		}
+
+		IO::outb(PIT_CTL, SELECT_CHANNEL0 | ACCESS_LO | ACCESS_HI | MODE_RATE_GENERATOR);
+		IO::outb(TIMER0_CTL, (s_ticks_per_ms >> 0) & 0xFF);
+		IO::outb(TIMER0_CTL, (s_ticks_per_ms >> 8) & 0xFF);
+	}
+
 }

kernel/kernel/Timer/Timer.cpp

@@ -69,6 +69,11 @@ namespace Kernel
 		return m_timer->time_since_boot();
 	}
 
+	void SystemTimer::pre_scheduler_sleep_ns(uint64_t ns)
+	{
+		return m_timer->pre_scheduler_sleep_ns(ns);
+	}
+
 	void SystemTimer::sleep_ns(uint64_t ns) const
 	{
 		if (ns == 0)
@@ -76,10 +81,6 @@ namespace Kernel
 
 		const uint64_t wake_time_ns = ns_since_boot() + ns;
 		Processor::scheduler().block_current_thread(nullptr, wake_time_ns);
-
-		//const uint64_t current_time_ns = ns_since_boot();
-		//if (current_time_ns < wake_time_ns)
-		//	dwarnln("sleep woke {} ms too soon", BAN::Math::div_round_up<uint64_t>(wake_time_ns - current_time_ns, 1'000'000));
 	}
 
 	timespec SystemTimer::real_time() const

kernel/kernel/kernel.cpp

@@ -1,4 +1,5 @@
 #include <kernel/ACPI/ACPI.h>
+#include <kernel/APIC.h>
 #include <kernel/Arch.h>
 #include <kernel/BootInfo.h>
 #include <kernel/Debug.h>
@@ -167,6 +168,12 @@ extern "C" void kernel_main(uint32_t boot_magic, uint32_t boot_info)
 	Random::initialize();
 	dprintln("RNG initialized");
 
+	if (InterruptController::get().is_using_apic())
+	{
+		SystemTimer::get().dont_invoke_scheduler();
+		static_cast<APIC&>(InterruptController::get()).initialize_timer();
+	}
+
 	Processor::wait_until_processors_ready();
 	MUST(Processor::scheduler().initialize());