Kernel: Don't read CR registers on every exception

kernel/arch/i686/interrupts.S

@@ -27,14 +27,10 @@
 
 isr_stub:
 	intr_header 12
-	movl %cr0, %eax; pushl %eax
-	movl %cr2, %eax; pushl %eax
-	movl %cr3, %eax; pushl %eax
-	movl %cr4, %eax; pushl %eax
 
-	movl 48(%esp), %edi // isr number
-	movl 52(%esp), %esi // error code
-	leal 56(%esp), %edx // interrupt stack ptr
+	movl 32(%esp), %edi // isr number
+	movl 36(%esp), %esi // error code
+	leal 40(%esp), %edx // interrupt stack ptr
 	movl %esp,     %ecx // register ptr
 
 	# stack frame for stack trace
@@ -52,7 +48,7 @@ isr_stub:
 	call cpp_isr_handler
 
 	movl %ebp, %esp
-	addl $24, %esp
+	addl $8, %esp
 
 	intr_footer 12
 	addl $8, %esp

kernel/arch/x86_64/interrupts.S

@@ -44,17 +44,12 @@
 
 isr_stub:
 	intr_header 24
-	movq %cr0, %rax; pushq %rax
-	movq %cr2, %rax; pushq %rax
-	movq %cr3, %rax; pushq %rax
-	movq %cr4, %rax; pushq %rax
 
-	movq 152(%rsp), %rdi	// isr number
-	movq 160(%rsp), %rsi	// error code
-	leaq 168(%rsp), %rdx	// interrupt stack ptr
-	movq %rsp, %rcx			// register ptr
+	movq 120(%rsp), %rdi // isr number
+	movq 128(%rsp), %rsi // error code
+	leaq 136(%rsp), %rdx // interrupt stack ptr
+	movq %rsp, %rcx      // register ptr
 	call cpp_isr_handler
-	addq $32, %rsp
 
 	intr_footer 24
 	addq $16, %rsp

kernel/include/kernel/CPUID.h

@@ -83,5 +83,6 @@ namespace CPUID
 	bool has_1gib_pages();
 	bool has_invariant_tsc();
 	uint64_t get_tsc_frequency();
+	bool has_kvm_pvclock();
 
 }

kernel/include/kernel/Timer/Timer.h

@@ -2,6 +2,7 @@
 
 #include <BAN/UniqPtr.h>
 #include <BAN/Vector.h>
+#include <kernel/Memory/DMARegion.h>
 #include <kernel/Timer/RTC.h>
 
 #include <time.h>
@@ -69,12 +70,14 @@ namespace Kernel
 		void initialize_timers();
 
 		void initialize_invariant_tsc();
+		void initialize_pvclock();
 
 	private:
 		enum class TSCType
 		{
 			None,
 			Invariant,
+			PVClock,
 		};
 
 		uint64_t m_boot_time { 0 };
@@ -90,6 +93,7 @@ namespace Kernel
 			} invariant;
 		} m_tsc_info;
 
+		BAN::UniqPtr<DMARegion> m_tsc_page;
 		uint64_t m_tsc_update_ns { 0 };
 	};
 

kernel/kernel/CPUID.cpp

@@ -97,6 +97,16 @@ namespace CPUID
 		return static_cast<uint64_t>(buffer[2]) * buffer[1] / buffer[0];
 	}
 
+	bool has_kvm_pvclock()
+	{
+		uint32_t buffer[4] {};
+		get_cpuid(0x40000000, buffer);
+		if (buffer[1] != 0x4B4D564B || buffer[2] != 0x564B4D56 || buffer[3] != 0x4D)
+			return false;
+		get_cpuid(0x40000001, buffer);
+		return buffer[0] & (1 << 3);
+	}
+
 	const char* feature_string_ecx(uint32_t feat)
 	{
 		switch (feat)

kernel/kernel/IDT.cpp

@@ -24,11 +24,6 @@ namespace Kernel
 #if ARCH(x86_64)
 	struct Registers
 	{
-		uint64_t cr4;
-		uint64_t cr3;
-		uint64_t cr2;
-		uint64_t cr0;
-
 		uint64_t r15;
 		uint64_t r14;
 		uint64_t r13;
@@ -49,11 +44,6 @@ namespace Kernel
 #elif ARCH(i686)
 	struct Registers
 	{
-		uint32_t cr4;
-		uint32_t cr3;
-		uint32_t cr2;
-		uint32_t cr0;
-
 		uint32_t edi;
 		uint32_t esi;
 		uint32_t ebp;
@@ -226,8 +216,11 @@ namespace Kernel
 					return;
 				}
 
+				uintptr_t cr2;
+				asm volatile("mov %%cr2, %0" : "=r"(cr2));
+
 				Processor::set_interrupt_state(InterruptState::Enabled);
-				auto result = Process::current().allocate_page_for_demand_paging(regs->cr2, page_fault_error.write, page_fault_error.instruction);
+				auto result = Process::current().allocate_page_for_demand_paging(cr2, page_fault_error.write, page_fault_error.instruction);
 				Processor::set_interrupt_state(InterruptState::Disabled);
 
 				if (result.is_error())
@@ -286,6 +279,15 @@ namespace Kernel
 			}
 		}
 
+		uintptr_t cr0, cr2, cr3, cr4;
+		asm volatile(
+			"mov %%cr0, %0;"
+			"mov %%cr2, %1;"
+			"mov %%cr3, %2;"
+			"mov %%cr4, %3;"
+			: "=r"(cr0), "=r"(cr2), "=r"(cr3), "=r"(cr4)
+		);
+
 		Debug::s_debug_lock.lock();
 
 		if (PageTable::current().get_page_flags(interrupt_stack->ip & PAGE_ADDR_MASK) & PageTable::Flags::Present)
@@ -323,7 +325,7 @@ namespace Kernel
 			regs->r8, regs->r9, regs->r10, regs->r11,
 			regs->r12, regs->r13, regs->r14, regs->r15,
 			interrupt_stack->ip, interrupt_stack->flags,
-			regs->cr0, regs->cr2, regs->cr3, regs->cr4
+			cr0, cr2, cr3, cr4
 		);
 #elif ARCH(i686)
 		dwarnln(
@@ -337,7 +339,7 @@ namespace Kernel
 			regs->eax, regs->ebx, regs->ecx, regs->edx,
 			interrupt_stack->sp, regs->ebp, regs->edi, regs->esi,
 			interrupt_stack->ip, interrupt_stack->flags,
-			regs->cr0, regs->cr2, regs->cr3, regs->cr4
+			cr0, cr2, cr3, cr4
 		);
 #endif
 		if (isr == ISR::PageFault)
@@ -385,11 +387,11 @@ namespace Kernel
 				break;
 			case ISR::PageFault:
 				signal_info.si_signo = SIGSEGV;
-				if (PageTable::current().get_page_flags(regs->cr2 & PAGE_ADDR_MASK) & PageTable::Flags::Present)
+				if (PageTable::current().get_page_flags(cr2 & PAGE_ADDR_MASK) & PageTable::Flags::Present)
 					signal_info.si_code = SEGV_ACCERR;
 				else
 					signal_info.si_code = SEGV_MAPERR;
-				signal_info.si_addr = reinterpret_cast<void*>(regs->cr2);
+				signal_info.si_addr = reinterpret_cast<void*>(cr2);
 				break;
 			default:
 				dwarnln("Unhandled exception");

kernel/kernel/Timer/Timer.cpp

@@ -11,6 +11,18 @@ namespace Kernel
 
 	static SystemTimer* s_instance = nullptr;
 
+	struct pvclock_vcpu_time_info
+	{
+		uint32_t version;
+		uint32_t pad0;
+		uint64_t tsc_timestamp;
+		uint64_t system_time;
+		uint32_t tsc_to_system_mul;
+		int8_t tsc_shift;
+		uint8_t flags;
+		uint8_t pad[2];
+	};
+
 	void SystemTimer::initialize()
 	{
 		ASSERT(s_instance == nullptr);
@@ -59,6 +71,8 @@ namespace Kernel
 
 	void SystemTimer::initialize_tsc()
 	{
+		if (CPUID::has_kvm_pvclock())
+			return initialize_pvclock();
 		if (CPUID::has_invariant_tsc())
 			return initialize_invariant_tsc();
 		dwarnln("No supported TSC based timers available");
@@ -108,6 +122,39 @@ namespace Kernel
 		dprintln("Initialized invariant TSC ({} Hz)", tsc_freq);
 	}
 
+	static pvclock_vcpu_time_info read_pvclock_safe(vaddr_t pvclock_vaddr)
+	{
+		for (;;)
+		{
+			const volatile auto& pvclock = *reinterpret_cast<const volatile pvclock_vcpu_time_info*>(pvclock_vaddr);
+
+			const auto version = pvclock.version;
+			if (version & 1)
+				continue;
+
+			pvclock_vcpu_time_info copy;
+			memcpy(&copy, const_cast<const pvclock_vcpu_time_info*>(&pvclock), sizeof(pvclock_vcpu_time_info));
+
+			if (pvclock.version == version)
+				return copy;
+		}
+	}
+
+	void SystemTimer::initialize_pvclock()
+	{
+		m_tsc_page = MUST(DMARegion::create(sizeof(pvclock_vcpu_time_info), PageTable::MemoryType::Normal));
+		memset(reinterpret_cast<void*>(m_tsc_page->vaddr()), 0, sizeof(pvclock_vcpu_time_info));
+
+		const uint32_t paddr_hi = m_tsc_page->paddr() >> 32;
+		const uint32_t paddr_lo = m_tsc_page->paddr() & 0xFFFFFFFF;
+		asm volatile("wrmsr" :: "d"(paddr_hi), "a"(paddr_lo | 1), "c"(0x4b564d01));
+
+		m_tsc_type = TSCType::PVClock;
+		Processor::initialize_tsc(m_boot_time);
+
+		dprintln("Initialized pvclock");
+	}
+
 	void SystemTimer::update_tsc()
 	{
 		if (m_tsc_type == TSCType::None)
@@ -137,6 +184,12 @@ namespace Kernel
 					.shift = m_tsc_info.invariant.shift,
 					.mult  = m_tsc_info.invariant.mult,
 				};
+			case TSCType::PVClock:
+				const auto pvclock = read_pvclock_safe(m_tsc_page->vaddr());
+				return {
+					.shift = pvclock.tsc_shift,
+					.mult  = pvclock.tsc_to_system_mul,
+				};
 		}
 		ASSERT_NOT_REACHED();
 	}

script/qemu.sh

@@ -38,7 +38,7 @@ USB_ARGS='-device qemu-xhci -device usb-kbd,port=1 -device usb-hub,port=2 -devic
 SOUND_ARGS='-device intel-hda -device hda-output'
 
 if [[ $@ == *"-accel kvm"* ]]; then
-	CPU_ARGS='-cpu host,migratable=off'
+	CPU_ARGS='-cpu host'
 fi
 
 qemu-system-$QEMU_ARCH \