Kernel: Implement WriteCombining memory

This makes framebuffer much faster on real hardware

kernel/arch/i686/PageTable.cpp

@@ -20,6 +20,7 @@ namespace Kernel
 	static PageTable* s_kernel = nullptr;
 	static bool s_has_nxe = false;
 	static bool s_has_pge = false;
+	static bool s_has_pat = false;
 
 	static paddr_t s_global_pdpte = 0;
 
@@ -32,8 +33,6 @@ namespace Kernel
 			result |= Flags::Execute;
 		if (entry & Flags::Reserved)
 			result |= Flags::Reserved;
-		if (entry & Flags::CacheDisable)
-			result |= Flags::CacheDisable;
 		if (entry & Flags::UserSupervisor)
 			result |= Flags::UserSupervisor;
 		if (entry & Flags::ReadWrite)
@@ -51,6 +50,9 @@ namespace Kernel
 		if (CPUID::has_pge())
 			s_has_pge = true;
 
+		if (CPUID::has_pat())
+			s_has_pat = true;
+
 		ASSERT(s_kernel == nullptr);
 		s_kernel = new PageTable();
 		ASSERT(s_kernel);
@@ -82,6 +84,17 @@ namespace Kernel
 			);
 		}
 
+		if (s_has_pat)
+		{
+			asm volatile(
+				"movl $0x277, %%ecx;"
+				"rdmsr;"
+				"movw $0x0401, %%dx;"
+				"wrmsr;"
+				::: "eax", "ecx", "edx", "memory"
+			);
+		}
+
 		// enable write protect
 		asm volatile(
 			"movl %%cr0, %%eax;"
@@ -316,7 +329,7 @@ namespace Kernel
 			unmap_page(page * PAGE_SIZE);
 	}
 
-	void PageTable::map_page_at(paddr_t paddr, vaddr_t vaddr, flags_t flags)
+	void PageTable::map_page_at(paddr_t paddr, vaddr_t vaddr, flags_t flags, MemoryType memory_type)
 	{
 		ASSERT(vaddr);
 		ASSERT(vaddr != fast_page());
@@ -338,8 +351,14 @@ namespace Kernel
 			extra_flags |= 1ull << 63;
 		if (flags & Flags::Reserved)
 			extra_flags |= Flags::Reserved;
-		if (flags & Flags::CacheDisable)
+
-			extra_flags |= Flags::CacheDisable;
+		if (s_has_pat)
+		{
+			if (memory_type == MemoryType::WriteCombining)
+				extra_flags |= (1ull << 7);
+			if (memory_type == MemoryType::WriteThrough)
+				extra_flags |= (1ull << 7) | (1ull << 3);
+		}
 
 		// NOTE: we add present here, since it has to be available in higher level structures
 		flags_t uwr_flags = (flags & (Flags::UserSupervisor | Flags::ReadWrite)) | Flags::Present;
@@ -367,7 +386,7 @@ namespace Kernel
 		invalidate(vaddr);
 	}
 
-	void PageTable::map_range_at(paddr_t paddr, vaddr_t vaddr, size_t size, flags_t flags)
+	void PageTable::map_range_at(paddr_t paddr, vaddr_t vaddr, size_t size, flags_t flags, MemoryType memory_type)
 	{
 		ASSERT(vaddr);
 		ASSERT(paddr % PAGE_SIZE == 0);
@@ -377,7 +396,7 @@ namespace Kernel
 
 		SpinLockGuard _(m_lock);
 		for (size_t page = 0; page < page_count; page++)
-			map_page_at(paddr + page * PAGE_SIZE, vaddr + page * PAGE_SIZE, flags);
+			map_page_at(paddr + page * PAGE_SIZE, vaddr + page * PAGE_SIZE, flags, memory_type);
 	}
 
 	uint64_t PageTable::get_page_data(vaddr_t vaddr) const

kernel/arch/x86_64/PageTable.cpp

@@ -56,8 +56,6 @@ namespace Kernel
 			result |= Flags::Execute;
 		if (entry & Flags::Reserved)
 			result |= Flags::Reserved;
-		if (entry & Flags::CacheDisable)
-			result |= Flags::CacheDisable;
 		if (entry & Flags::UserSupervisor)
 			result |= Flags::UserSupervisor;
 		if (entry & Flags::ReadWrite)
@@ -106,6 +104,15 @@ namespace Kernel
 			);
 		}
 
+		// 64-bit always has PAT, set PAT4 = WC, PAT5 = WT
+		asm volatile(
+			"movl $0x277, %%ecx;"
+			"rdmsr;"
+			"movw $0x0401, %%dx;"
+			"wrmsr;"
+			::: "eax", "ecx", "edx", "memory"
+		);
+
 		// enable write protect
 		asm volatile(
 			"movq %%cr0, %%rax;"
@@ -367,7 +374,7 @@ namespace Kernel
 			unmap_page(page * PAGE_SIZE);
 	}
 
-	void PageTable::map_page_at(paddr_t paddr, vaddr_t vaddr, flags_t flags)
+	void PageTable::map_page_at(paddr_t paddr, vaddr_t vaddr, flags_t flags, MemoryType memory_type)
 	{
 		ASSERT(vaddr);
 		ASSERT(vaddr != fast_page());
@@ -393,8 +400,11 @@ namespace Kernel
 			extra_flags |= 1ull << 63;
 		if (flags & Flags::Reserved)
 			extra_flags |= Flags::Reserved;
-		if (flags & Flags::CacheDisable)
+
-			extra_flags |= Flags::CacheDisable;
+		if (memory_type == MemoryType::WriteCombining)
+			extra_flags |= (1ull << 7);
+		if (memory_type == MemoryType::WriteThrough)
+			extra_flags |= (1ull << 7) | (1ull << 3);
 
 		// NOTE: we add present here, since it has to be available in higher level structures
 		flags_t uwr_flags = (flags & (Flags::UserSupervisor | Flags::ReadWrite)) | Flags::Present;
@@ -434,7 +444,7 @@ namespace Kernel
 		invalidate(vaddr);
 	}
 
-	void PageTable::map_range_at(paddr_t paddr, vaddr_t vaddr, size_t size, flags_t flags)
+	void PageTable::map_range_at(paddr_t paddr, vaddr_t vaddr, size_t size, flags_t flags, MemoryType memory_type)
 	{
 		ASSERT(is_canonical(vaddr));
 
@@ -446,7 +456,7 @@ namespace Kernel
 
 		SpinLockGuard _(m_lock);
 		for (size_t page = 0; page < page_count; page++)
-			map_page_at(paddr + page * PAGE_SIZE, vaddr + page * PAGE_SIZE, flags);
+			map_page_at(paddr + page * PAGE_SIZE, vaddr + page * PAGE_SIZE, flags, memory_type);
 	}
 
 	uint64_t PageTable::get_page_data(vaddr_t vaddr) const

kernel/include/kernel/CPUID.h

@@ -79,5 +79,6 @@ namespace CPUID
 	bool is_64_bit();
 	bool has_nxe();
 	bool has_pge();
+	bool has_pat();
 
 }

kernel/include/kernel/Memory/PageTable.h

@@ -29,12 +29,17 @@ namespace Kernel
 			Present			= (1 << 0),
 			ReadWrite		= (1 << 1),
 			UserSupervisor	= (1 << 2),
-			CacheDisable	= (1 << 4),
 			Reserved		= (1 << 9),
 
 			Execute			= (1 << 15),
 			Used = Present | Reserved,
 		};
+		enum MemoryType
+		{
+			Normal,
+			WriteCombining,
+			WriteThrough,
+		};
 
 	public:
 		static void initialize();
@@ -93,8 +98,8 @@ namespace Kernel
 		void unmap_page(vaddr_t);
 		void unmap_range(vaddr_t, size_t bytes);
 
-		void map_range_at(paddr_t, vaddr_t, size_t bytes, flags_t);
+		void map_range_at(paddr_t, vaddr_t, size_t bytes, flags_t, MemoryType = MemoryType::Normal);
-		void map_page_at(paddr_t, vaddr_t, flags_t);
+		void map_page_at(paddr_t, vaddr_t, flags_t, MemoryType = MemoryType::Normal);
 
 		paddr_t physical_address_of(vaddr_t) const;
 		flags_t get_page_flags(vaddr_t) const;

kernel/kernel/CPUID.cpp

@@ -57,6 +57,13 @@ namespace CPUID
 		return edx & CPUID::EDX_PGE;
 	}
 
+	bool has_pat()
+	{
+		uint32_t ecx, edx;
+		get_features(ecx, edx);
+		return edx & CPUID::EDX_PAT;
+	}
+
 	const char* feature_string_ecx(uint32_t feat)
 	{
 		switch (feat)

kernel/kernel/Device/FramebufferDevice.cpp

@@ -68,7 +68,8 @@ namespace Kernel
 			m_video_memory_paddr & PAGE_ADDR_MASK,
 			m_video_memory_vaddr,
 			video_memory_pages * PAGE_SIZE,
-			PageTable::Flags::ReadWrite | PageTable::Flags::Present
+			PageTable::Flags::ReadWrite | PageTable::Flags::Present,
+			PageTable::WriteCombining
 		);
 
 		m_video_buffer = TRY(VirtualRange::create_to_vaddr_range(

kernel/kernel/Memory/DMARegion.cpp

@@ -26,7 +26,7 @@ namespace Kernel
 		vaddr_guard.disable();
 		paddr_guard.disable();
 
-		PageTable::kernel().map_range_at(paddr, vaddr, size, PageTable::Flags::CacheDisable | PageTable::Flags::ReadWrite | PageTable::Flags::Present);
+		PageTable::kernel().map_range_at(paddr, vaddr, size, PageTable::Flags::ReadWrite | PageTable::Flags::Present);
 
 		return BAN::UniqPtr<DMARegion>::adopt(region_ptr);
 	}

kernel/kernel/PCI.cpp

@@ -652,7 +652,8 @@ namespace Kernel::PCI
 		m_vaddr = PageTable::kernel().reserve_free_contiguous_pages(needed_pages, KERNEL_OFFSET);
 		if (m_vaddr == 0)
 			return BAN::Error::from_errno(ENOMEM);
-		PageTable::kernel().map_range_at(m_paddr, m_vaddr, m_size, PageTable::Flags::CacheDisable | PageTable::Flags::ReadWrite | PageTable::Flags::Present);
+
+		PageTable::kernel().map_range_at(m_paddr, m_vaddr, m_size, PageTable::Flags::ReadWrite | PageTable::Flags::Present);
 
 		return {};
 	}