Kernel: Rewrite paging and AP initialization

Initial step of paging now just prepares fast page for heap, actual page
table initialization happens after heap is initialized which allows
x86_64 to never depend on kmalloc for pages.

Processor's stacks are now also spawned with PMM/VMM allocated stacks
instead of kmalloc identity mapped.

kernel/arch/i686/PageTable.cpp

@@ -16,6 +16,8 @@ extern uint8_t g_kernel_writable_end[];
 extern uint8_t g_userspace_start[];
 extern uint8_t g_userspace_end[];
 
+extern uint64_t g_boot_fast_page_pt[];
+
 namespace Kernel
 {
 
@@ -24,7 +26,7 @@ namespace Kernel
 	constexpr uint64_t s_page_flag_mask = 0x8000000000000FFF;
 	constexpr uint64_t s_page_addr_mask = ~s_page_flag_mask;
 
-	static bool s_is_post_heap_done = false;
+	static bool s_is_initialized = false;
 
 	static PageTable* s_kernel = nullptr;
 	static bool s_has_nxe = false;
@@ -33,6 +35,28 @@ namespace Kernel
 
 	static paddr_t s_global_pdpte = 0;
 
+	static uint64_t* s_fast_page_pt { nullptr };
+
+	static uint64_t* allocate_zeroed_page_aligned_page()
+	{
+		void* page = kmalloc(PAGE_SIZE, PAGE_SIZE, true);
+		ASSERT(page);
+		memset(page, 0, PAGE_SIZE);
+		return (uint64_t*)page;
+	}
+
+	template<typename T>
+	static paddr_t V2P(const T vaddr)
+	{
+		return (vaddr_t)vaddr - KERNEL_OFFSET + g_boot_info.kernel_paddr;
+	}
+
+	template<typename T>
+	static uint64_t* P2V(const T paddr)
+	{
+		return reinterpret_cast<uint64_t*>(reinterpret_cast<paddr_t>(paddr) - g_boot_info.kernel_paddr + KERNEL_OFFSET);
+	}
+
 	static inline PageTable::flags_t parse_flags(uint64_t entry)
 	{
 		using Flags = PageTable::Flags;
@@ -51,31 +75,22 @@ namespace Kernel
 		return result;
 	}
 
-	void PageTable::initialize_pre_heap()
+	void PageTable::initialize_fast_page()
+	{
+		s_fast_page_pt = g_boot_fast_page_pt;
+	}
+
+	static void detect_cpu_features()
 	{
 		if (CPUID::has_nxe())
 			s_has_nxe = true;
-
 		if (CPUID::has_pge())
 			s_has_pge = true;
-
 		if (CPUID::has_pat())
 			s_has_pat = true;
-
-		ASSERT(s_kernel == nullptr);
-		s_kernel = new PageTable();
-		ASSERT(s_kernel);
-
-		s_kernel->initialize_kernel();
-		s_kernel->initial_load();
 	}
 
-	void PageTable::initialize_post_heap()
+	void PageTable::enable_cpu_features()
-	{
-		s_is_post_heap_done = true;
-	}
-
-	void PageTable::initial_load()
 	{
 		if (s_has_nxe)
 		{
@@ -116,8 +131,56 @@ namespace Kernel
 			"movl %%eax, %%cr0;"
 			::: "rax"
 		);
+	}
 
-		load();
+	void PageTable::initialize_and_load()
+	{
+		detect_cpu_features();
+		enable_cpu_features();
+
+		ASSERT(s_kernel == nullptr);
+		s_kernel = new PageTable();
+		ASSERT(s_kernel);
+
+		auto* pdpt = allocate_zeroed_page_aligned_page();
+		ASSERT(pdpt);
+
+		s_kernel->m_highest_paging_struct = V2P(pdpt);
+		s_kernel->map_kernel_memory();
+
+		PageTable::with_fast_page(s_kernel->m_highest_paging_struct, [] {
+			s_global_pdpte = PageTable::fast_page_as_sized<paddr_t>(3);
+		});
+
+		// update fast page pt
+		{
+			constexpr vaddr_t vaddr = fast_page();
+			constexpr uint16_t pdpte = (vaddr >> 30) & 0x1FF;
+			constexpr uint16_t pde   = (vaddr >> 21) & 0x1FF;
+
+			const auto get_or_allocate_entry =
+				[](paddr_t table_paddr, uint16_t entry, uint64_t flags)
+				{
+					uint64_t* table = P2V(table_paddr);
+
+					if (!(table[entry] & Flags::Present))
+					{
+						auto* vaddr = allocate_zeroed_page_aligned_page();
+						ASSERT(vaddr);
+						table[entry] = V2P(vaddr);
+					}
+
+					table[entry] |= flags;
+
+					return table[entry] & s_page_addr_mask;
+				};
+
+			const paddr_t pdpt = s_kernel->m_highest_paging_struct;
+			const paddr_t pd = get_or_allocate_entry(pdpt, pdpte, Flags::Present);
+			s_fast_page_pt = P2V(get_or_allocate_entry(pd, pde, Flags::ReadWrite | Flags::Present));
+		}
+
+		s_kernel->load();
 	}
 
 	PageTable& PageTable::kernel()
@@ -131,40 +194,12 @@ namespace Kernel
 		return true;
 	}
 
-	static uint64_t* allocate_zeroed_page_aligned_page()
+	void PageTable::map_kernel_memory()
 	{
-		void* page = kmalloc(PAGE_SIZE, PAGE_SIZE, true);
-		ASSERT(page);
-		memset(page, 0, PAGE_SIZE);
-		return (uint64_t*)page;
-	}
-
-	template<typename T>
-	static paddr_t V2P(const T vaddr)
-	{
-		return (vaddr_t)vaddr - KERNEL_OFFSET + g_boot_info.kernel_paddr;
-	}
-
-	template<typename T>
-	static uint64_t* P2V(const T paddr)
-	{
-		return reinterpret_cast<uint64_t*>(reinterpret_cast<paddr_t>(paddr) - g_boot_info.kernel_paddr + KERNEL_OFFSET);
-	}
-
-	void PageTable::initialize_kernel()
-	{
-		ASSERT(s_global_pdpte == 0);
-		s_global_pdpte = V2P(allocate_zeroed_page_aligned_page());
-
-		map_kernel_memory();
-
-		prepare_fast_page();
-
 		// Map (phys_kernel_start -> phys_kernel_end) to (virt_kernel_start -> virt_kernel_end)
-		ASSERT((vaddr_t)g_kernel_start % PAGE_SIZE == 0);
 		map_range_at(
 			V2P(g_kernel_start),
-			(vaddr_t)g_kernel_start,
+			reinterpret_cast<vaddr_t>(g_kernel_start),
 			g_kernel_end - g_kernel_start,
 			Flags::Present
 		);
@@ -172,7 +207,7 @@ namespace Kernel
 		// Map executable kernel memory as executable
 		map_range_at(
 			V2P(g_kernel_execute_start),
-			(vaddr_t)g_kernel_execute_start,
+			reinterpret_cast<vaddr_t>(g_kernel_execute_start),
 			g_kernel_execute_end - g_kernel_execute_start,
 			Flags::Execute | Flags::Present
 		);
@@ -180,7 +215,7 @@ namespace Kernel
 		// Map writable kernel memory as writable
 		map_range_at(
 			V2P(g_kernel_writable_start),
-			(vaddr_t)g_kernel_writable_start,
+			reinterpret_cast<vaddr_t>(g_kernel_writable_start),
 			g_kernel_writable_end - g_kernel_writable_start,
 			Flags::ReadWrite | Flags::Present
 		);
@@ -188,70 +223,34 @@ namespace Kernel
 		// Map userspace memory
 		map_range_at(
 			V2P(g_userspace_start),
-			(vaddr_t)g_userspace_start,
+			reinterpret_cast<vaddr_t>(g_userspace_start),
 			g_userspace_end - g_userspace_start,
 			Flags::Execute | Flags::UserSupervisor | Flags::Present
 		);
 	}
 
-	void PageTable::prepare_fast_page()
-	{
-		constexpr uint64_t pdpte = (fast_page() >> 30) & 0x1FF;
-		constexpr uint64_t pde   = (fast_page() >> 21) & 0x1FF;
-		constexpr uint64_t pte   = (fast_page() >> 12) & 0x1FF;
-
-		const uint64_t* pdpt = P2V(m_highest_paging_struct);
-		ASSERT(pdpt[pdpte] & Flags::Present);
-
-		uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
-		ASSERT(!(pd[pde] & Flags::Present));
-		pd[pde] = V2P(allocate_zeroed_page_aligned_page()) | Flags::ReadWrite | Flags::Present;
-
-		uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
-		ASSERT(pt[pte] == 0);
-		pt[pte] = Flags::Reserved;
-	}
-
 	void PageTable::map_fast_page(paddr_t paddr)
 	{
-		ASSERT(s_kernel);
+		ASSERT(paddr && paddr % PAGE_SIZE == 0);
-		ASSERT(paddr);
-		ASSERT(paddr % PAGE_SIZE == 0);
 
+		ASSERT(s_fast_page_pt);
 		ASSERT(s_fast_page_lock.current_processor_has_lock());
 
-		constexpr uint64_t pdpte = (fast_page() >> 30) & 0x1FF;
+		ASSERT(!(*s_fast_page_pt & Flags::Present));
-		constexpr uint64_t pde   = (fast_page() >> 21) & 0x1FF;
+		s_fast_page_pt[0] = paddr | Flags::ReadWrite | Flags::Present;
-		constexpr uint64_t pte   = (fast_page() >> 12) & 0x1FF;
 
-		uint64_t* pdpt = P2V(s_kernel->m_highest_paging_struct);
+		asm volatile("invlpg (%0)" :: "r"(fast_page()));
-		uint64_t* pd   = P2V(pdpt[pdpte] & s_page_addr_mask);
-		uint64_t* pt   = P2V(pd[pde] & s_page_addr_mask);
-
-		ASSERT(!(pt[pte] & Flags::Present));
-		pt[pte] = paddr | Flags::ReadWrite | Flags::Present;
-
-		asm volatile("invlpg (%0)" :: "r"(fast_page()) : "memory");
 	}
 
 	void PageTable::unmap_fast_page()
 	{
-		ASSERT(s_kernel);
+		ASSERT(s_fast_page_pt);
-
 		ASSERT(s_fast_page_lock.current_processor_has_lock());
 
-		constexpr uint64_t pdpte = (fast_page() >> 30) & 0x1FF;
+		ASSERT((*s_fast_page_pt & Flags::Present));
-		constexpr uint64_t pde   = (fast_page() >> 21) & 0x1FF;
+		s_fast_page_pt[0] = 0;
-		constexpr uint64_t pte   = (fast_page() >> 12) & 0x1FF;
 
-		uint64_t* pdpt = P2V(s_kernel->m_highest_paging_struct);
+		asm volatile("invlpg (%0)" :: "r"(fast_page()));
-		uint64_t* pd   = P2V(pdpt[pdpte] & s_page_addr_mask);
-		uint64_t* pt   = P2V(pd[pde] & s_page_addr_mask);
-
-		ASSERT(pt[pte] & Flags::Present);
-		pt[pte] = Flags::Reserved;
-
-		asm volatile("invlpg (%0)" :: "r"(fast_page()) : "memory");
 	}
 
 	BAN::ErrorOr<PageTable*> PageTable::create_userspace()
@@ -260,25 +259,23 @@ namespace Kernel
 		PageTable* page_table = new PageTable;
 		if (page_table == nullptr)
 			return BAN::Error::from_errno(ENOMEM);
-		page_table->map_kernel_memory();
+
-		return page_table;
+		uint64_t* pdpt = allocate_zeroed_page_aligned_page();
+		if (pdpt == nullptr)
+		{
+			delete page_table;
+			return BAN::Error::from_errno(ENOMEM);
 		}
 
-	void PageTable::map_kernel_memory()
+		page_table->m_highest_paging_struct = V2P(pdpt);
-	{
-		ASSERT(s_kernel);
-		ASSERT(s_global_pdpte);
 
-		ASSERT(m_highest_paging_struct == 0);
-		m_highest_paging_struct = V2P(kmalloc(32, 32, true));
-		ASSERT(m_highest_paging_struct);
-
-		uint64_t* pdpt = P2V(m_highest_paging_struct);
 		pdpt[0] = 0;
 		pdpt[1] = 0;
 		pdpt[2] = 0;
 		pdpt[3] = s_global_pdpte | Flags::Present;
 		static_assert(KERNEL_OFFSET == 0xC0000000);
+
+		return page_table;
 	}
 
 	PageTable::~PageTable()
@@ -318,7 +315,7 @@ namespace Kernel
 		const bool is_userspace = (vaddr < KERNEL_OFFSET);
 		if (is_userspace && this != &PageTable::current())
 			;
-		else if (pages <= 32 || !s_is_post_heap_done)
+		else if (pages <= 32 || !s_is_initialized)
 		{
 			for (size_t i = 0; i < pages; i++, vaddr += PAGE_SIZE)
 				asm volatile("invlpg (%0)" :: "r"(vaddr));

kernel/arch/i686/boot.S

@@ -98,8 +98,7 @@ bananboot_end:
 boot_pdpt:
 	.long V2P(boot_pd) + (PG_PRESENT)
 	.long 0
-	.quad 0
+	.skip 2 * 8
-	.quad 0
 	.long V2P(boot_pd) + (PG_PRESENT)
 	.long 0
 .align 4096
@@ -112,13 +111,16 @@ boot_pd:
 	.endr
 boot_pts:
 	.set i, 0
-	.rept 512
+	.rept 511
 		.rept 512
 			.long i + (PG_READ_WRITE | PG_PRESENT)
 			.long 0
 			.set i, i + 0x1000
 		.endr
 	.endr
+.global g_boot_fast_page_pt
+g_boot_fast_page_pt:
+	.skip 512 * 8
 
 boot_gdt:
 	.quad 0x0000000000000000 # null descriptor
@@ -274,7 +276,7 @@ system_halt:
 	jmp 1b
 
 
-#define AP_V2P(vaddr) ((vaddr) - ap_trampoline + 0xF000)
+#define AP_REL(vaddr) ((vaddr) - ap_trampoline + 0xF000)
 
 .section .ap_init, "ax"
 
@@ -284,21 +286,27 @@ ap_trampoline:
 	jmp 1f
 
 .align 8
-ap_stack_ptr:
+ap_stack_paddr:
+	.skip 4
+ap_stack_vaddr:
+	.skip 4
+ap_prepare_paging:
+	.skip 4
+ap_page_table:
+	.skip 4
+ap_ready:
 	.skip 4
-ap_stack_loaded:
-	.skip 1
 
 1:	cli; cld
-	ljmpl $0x00, $AP_V2P(ap_cs_clear)
+	ljmpl $0x00, $AP_REL(ap_cs_clear)
 
 ap_cs_clear:
 	# load ap gdt and enter protected mode
-	lgdt AP_V2P(ap_gdtr)
+	lgdt AP_REL(ap_gdtr)
 	movl %cr0, %eax
 	orb $1, %al
 	movl %eax, %cr0
-	ljmpl $0x08, $AP_V2P(ap_protected_mode)
+	ljmpl $0x08, $AP_REL(ap_protected_mode)
 
 .code32
 ap_protected_mode:
@@ -307,8 +315,7 @@ ap_protected_mode:
 	movw %ax, %ss
 	movw %ax, %es
 
-	movl AP_V2P(ap_stack_ptr), %esp
+	movl AP_REL(ap_stack_paddr), %esp
-	movb $1, AP_V2P(ap_stack_loaded)
 
 	leal V2P(enable_sse),        %ecx; call *%ecx
 	leal V2P(enable_tsc),        %ecx; call *%ecx
@@ -316,24 +323,28 @@ ap_protected_mode:
 
 	# load boot gdt and enter long mode
 	lgdt V2P(boot_gdtr)
-	ljmpl $0x08, $AP_V2P(ap_flush_gdt)
+	ljmpl $0x08, $AP_REL(ap_flush_gdt)
 
 ap_flush_gdt:
-	# move stack pointer to higher half
+	movl $ap_higher_half, %ecx
-	movl %esp, %esp
-	addl $KERNEL_OFFSET, %esp
-
-	# jump to higher half
-	leal ap_higher_half, %ecx
 	jmp *%ecx
 
 ap_higher_half:
+	movl AP_REL(ap_prepare_paging), %eax
+	call *%eax
+
+	# load AP's initial values
+	movl AP_REL(ap_stack_vaddr), %esp
+	movl AP_REL(ap_page_table), %eax
+	movl $1, AP_REL(ap_ready)
+	movl %eax, %cr3
+
 	# clear rbp for stacktrace
 	xorl %ebp, %ebp
 
 1:	pause
 	cmpb $0, g_ap_startup_done
-	jz 1b
+	je 1b
 
 	lock incb g_ap_running_count
 

kernel/arch/i686/linker.ld

@@ -15,17 +15,17 @@ SECTIONS
 		*(.bananboot)
 		*(.text.*)
 	}
+	.ap_init ALIGN(4K) : AT(ADDR(.ap_init) - KERNEL_OFFSET)
+	{
+		g_ap_init_addr = .;
+		*(.ap_init)
+		g_kernel_execute_end = .;
+	}
 	.userspace ALIGN(4K) : AT(ADDR(.userspace) - KERNEL_OFFSET)
 	{
 		g_userspace_start = .;
 		*(.userspace)
 		g_userspace_end = .;
-		g_kernel_execute_end = .;
-	}
-	.ap_init ALIGN(4K) : AT(ADDR(.ap_init) - KERNEL_OFFSET)
-	{
-		g_ap_init_addr = .;
-		*(.ap_init)
 	}
 	.rodata ALIGN(4K) : AT(ADDR(.rodata) - KERNEL_OFFSET)
 	{

kernel/arch/x86_64/PageTable.cpp

@@ -2,7 +2,6 @@
 #include <kernel/CPUID.h>
 #include <kernel/Lock/SpinLock.h>
 #include <kernel/Memory/Heap.h>
-#include <kernel/Memory/kmalloc.h>
 #include <kernel/Memory/PageTable.h>
 
 extern uint8_t g_kernel_start[];
@@ -17,13 +16,15 @@ extern uint8_t g_kernel_writable_end[];
 extern uint8_t g_userspace_start[];
 extern uint8_t g_userspace_end[];
 
+extern uint64_t g_boot_fast_page_pt[];
+
 namespace Kernel
 {
 
 	SpinLock PageTable::s_fast_page_lock;
 
 	static constexpr vaddr_t s_hhdm_offset = 0xFFFF800000000000;
-	static bool s_is_post_heap_done = false;
+	static bool s_is_initialized = false;
 
 	constexpr uint64_t s_page_flag_mask = 0x8000000000000FFF;
 	constexpr uint64_t s_page_addr_mask = ~s_page_flag_mask;
@@ -35,6 +36,8 @@ namespace Kernel
 
 	static paddr_t s_global_pml4_entries[512] { 0 };
 
+	static uint64_t* s_fast_page_pt { nullptr };
+
 	static constexpr inline bool is_canonical(uintptr_t addr)
 	{
 		constexpr uintptr_t mask = 0xFFFF800000000000;
@@ -54,34 +57,6 @@ namespace Kernel
 		return addr;
 	}
 
-	struct FuncsKmalloc
-	{
-		static paddr_t allocate_zeroed_page_aligned_page()
-		{
-			void* page = kmalloc(PAGE_SIZE, PAGE_SIZE, true);
-			ASSERT(page);
-			memset(page, 0, PAGE_SIZE);
-			return kmalloc_paddr_of(reinterpret_cast<vaddr_t>(page)).value();
-		}
-
-		static void unallocate_page(paddr_t paddr)
-		{
-			kfree(reinterpret_cast<void*>(kmalloc_vaddr_of(paddr).value()));
-		}
-
-		static paddr_t V2P(vaddr_t vaddr)
-		{
-			return vaddr - KERNEL_OFFSET + g_boot_info.kernel_paddr;
-		}
-
-		static uint64_t* P2V(paddr_t paddr)
-		{
-			return reinterpret_cast<uint64_t*>(paddr - g_boot_info.kernel_paddr + KERNEL_OFFSET);
-		}
-	};
-
-	struct FuncsHHDM
-	{
 	static paddr_t allocate_zeroed_page_aligned_page()
 	{
 		const paddr_t paddr = Heap::get().take_free_page();
@@ -95,27 +70,14 @@ namespace Kernel
 		Heap::get().release_page(paddr);
 	}
 
-		static paddr_t V2P(vaddr_t vaddr)
-		{
-			ASSERT(vaddr >= s_hhdm_offset);
-			ASSERT(vaddr < KERNEL_OFFSET);
-			return vaddr - s_hhdm_offset;
-		}
-
 	static uint64_t* P2V(paddr_t paddr)
 	{
 		ASSERT(paddr != 0);
 		ASSERT(!BAN::Math::will_addition_overflow(paddr, s_hhdm_offset));
 		return reinterpret_cast<uint64_t*>(paddr + s_hhdm_offset);
 	}
-	};
 
-	static paddr_t   (*allocate_zeroed_page_aligned_page)() = &FuncsKmalloc::allocate_zeroed_page_aligned_page;
+	static PageTable::flags_t parse_flags(uint64_t entry)
-	static void      (*unallocate_page)(paddr_t)            = &FuncsKmalloc::unallocate_page;
-	static paddr_t   (*V2P)(vaddr_t)                        = &FuncsKmalloc::V2P;
-	static uint64_t* (*P2V)(paddr_t)                        = &FuncsKmalloc::P2V;
-
-	static inline PageTable::flags_t parse_flags(uint64_t entry)
 	{
 		using Flags = PageTable::Flags;
 
@@ -137,7 +99,7 @@ namespace Kernel
 	//   0: 4 KiB
 	//   1: 2 MiB
 	//   2: 1 GiB
-	static void init_map_hhdm_page(paddr_t pml4, paddr_t paddr, uint8_t page_size)
+	static void map_hhdm_page(paddr_t pml4, paddr_t paddr, uint8_t page_size)
 	{
 		ASSERT(0 <= page_size && page_size <= 2);
 
@@ -184,7 +146,7 @@ namespace Kernel
 		const uint64_t noexec_flag = s_has_nxe ? (static_cast<uint64_t>(1) << 63) : 0;
 
 		const paddr_t pdpt = get_or_allocate_entry(pml4, pml4e, noexec_flag);
-		s_global_pml4_entries[pml4e] = pdpt | hhdm_flags;
+		s_global_pml4_entries[pml4e] = pdpt | hhdm_flags | noexec_flag;
 
 		paddr_t lowest_paddr = pdpt;
 		uint16_t lowest_entry = pdpte;
@@ -207,23 +169,11 @@ namespace Kernel
 		});
 	}
 
-	static void init_map_hhdm(paddr_t pml4)
+	static void initialize_hhdm(paddr_t pml4)
 	{
 		for (const auto& entry : g_boot_info.memory_map_entries)
 		{
-			bool should_map = false;
+			if (entry.type != MemoryMapEntry::Type::Available)
-			switch (entry.type)
-			{
-				case MemoryMapEntry::Type::Available:
-					should_map = true;
-					break;
-				case MemoryMapEntry::Type::ACPIReclaim:
-				case MemoryMapEntry::Type::ACPINVS:
-				case MemoryMapEntry::Type::Reserved:
-					should_map = false;
-					break;
-			}
-			if (!should_map)
 				continue;
 
 			constexpr size_t one_gib = 1024 * 1024 * 1024;
@@ -235,156 +185,39 @@ namespace Kernel
 			{
 				if (s_has_gib && paddr % one_gib == 0 && paddr + one_gib <= entry_end)
 				{
-					init_map_hhdm_page(pml4, paddr, 2);
+					map_hhdm_page(pml4, paddr, 2);
 					paddr += one_gib;
 				}
 				else if (paddr % two_mib == 0 && paddr + two_mib <= entry_end)
 				{
-					init_map_hhdm_page(pml4, paddr, 1);
+					map_hhdm_page(pml4, paddr, 1);
 					paddr += two_mib;
 				}
 				else
 				{
-					init_map_hhdm_page(pml4, paddr, 0);
+					map_hhdm_page(pml4, paddr, 0);
 					paddr += PAGE_SIZE;
 				}
 			}
 		}
 	}
 
-	static paddr_t copy_page_from_kmalloc_to_heap(paddr_t kmalloc_paddr)
+	void PageTable::initialize_fast_page()
 	{
-		const paddr_t heap_paddr = Heap::get().take_free_page();
+		s_fast_page_pt = g_boot_fast_page_pt;
-		ASSERT(heap_paddr);
-
-		const vaddr_t kmalloc_vaddr = kmalloc_vaddr_of(kmalloc_paddr).value();
-
-		PageTable::with_fast_page(heap_paddr, [kmalloc_vaddr] {
-			memcpy(PageTable::fast_page_as_ptr(), reinterpret_cast<void*>(kmalloc_vaddr), PAGE_SIZE);
-		});
-
-		return heap_paddr;
 	}
 
-	static void copy_paging_structure_to_heap(uint64_t* old_table, uint64_t* new_table, int depth)
+	static void detect_cpu_features()
-	{
-		if (depth == 0)
-			return;
-
-		constexpr uint64_t page_flag_mask = 0x8000000000000FFF;
-		constexpr uint64_t page_addr_mask = ~page_flag_mask;
-
-		for (uint16_t index = 0; index < 512; index++)
-		{
-			const uint64_t old_entry = old_table[index];
-			if (old_entry == 0)
-			{
-				new_table[index] = 0;
-				continue;
-			}
-
-			const paddr_t old_paddr = old_entry & page_addr_mask;
-			const paddr_t new_paddr = copy_page_from_kmalloc_to_heap(old_paddr);
-			new_table[index] = new_paddr | (old_entry & page_flag_mask);
-
-			uint64_t* next_old_table = reinterpret_cast<uint64_t*>(old_paddr + s_hhdm_offset);
-			uint64_t* next_new_table = reinterpret_cast<uint64_t*>(new_paddr + s_hhdm_offset);
-			copy_paging_structure_to_heap(next_old_table, next_new_table, depth - 1);
-		}
-	}
-
-	static void free_kmalloc_paging_structure(uint64_t* table, int depth)
-	{
-		if (depth == 0)
-			return;
-
-		constexpr uint64_t page_flag_mask = 0x8000000000000FFF;
-		constexpr uint64_t page_addr_mask = ~page_flag_mask;
-
-		for (uint16_t index = 0; index < 512; index++)
-		{
-			const uint64_t entry = table[index];
-			if (entry == 0)
-				continue;
-
-			const paddr_t paddr = entry & page_addr_mask;
-
-			uint64_t* next_table = reinterpret_cast<uint64_t*>(paddr + s_hhdm_offset);
-			free_kmalloc_paging_structure(next_table, depth - 1);
-
-			kfree(reinterpret_cast<void*>(kmalloc_vaddr_of(paddr).value()));
-		}
-	}
-
-	void PageTable::initialize_pre_heap()
 	{
 		if (CPUID::has_nxe())
 			s_has_nxe = true;
-
 		if (CPUID::has_pge())
 			s_has_pge = true;
-
 		if (CPUID::has_1gib_pages())
 			s_has_gib = true;
-
-		ASSERT(s_kernel == nullptr);
-		s_kernel = new PageTable();
-		ASSERT(s_kernel);
-
-		s_kernel->m_highest_paging_struct = allocate_zeroed_page_aligned_page();
-		s_kernel->prepare_fast_page();
-		s_kernel->initialize_kernel();
-
-		for (auto pml4e : s_global_pml4_entries)
-			ASSERT(pml4e == 0);
-		const uint64_t* pml4 = P2V(s_kernel->m_highest_paging_struct);
-		s_global_pml4_entries[511] = pml4[511];
 	}
 
-	void PageTable::initialize_post_heap()
+	void PageTable::enable_cpu_features()
-	{
-		ASSERT(s_kernel);
-
-		init_map_hhdm(s_kernel->m_highest_paging_struct);
-
-		const paddr_t old_pml4_paddr = s_kernel->m_highest_paging_struct;
-		const paddr_t new_pml4_paddr = copy_page_from_kmalloc_to_heap(old_pml4_paddr);
-
-		uint64_t* old_pml4 = reinterpret_cast<uint64_t*>(kmalloc_vaddr_of(old_pml4_paddr).value());
-		uint64_t* new_pml4 = reinterpret_cast<uint64_t*>(new_pml4_paddr + s_hhdm_offset);
-
-		const paddr_t old_pdpt_paddr = old_pml4[511] & s_page_addr_mask;
-		const paddr_t new_pdpt_paddr = Heap::get().take_free_page();
-		ASSERT(new_pdpt_paddr);
-
-		uint64_t* old_pdpt = reinterpret_cast<uint64_t*>(old_pdpt_paddr + s_hhdm_offset);
-		uint64_t* new_pdpt = reinterpret_cast<uint64_t*>(new_pdpt_paddr + s_hhdm_offset);
-		copy_paging_structure_to_heap(old_pdpt, new_pdpt, 2);
-
-		new_pml4[511] = new_pdpt_paddr | (old_pml4[511] & s_page_flag_mask);
-		s_global_pml4_entries[511] = new_pml4[511];
-
-		s_kernel->m_highest_paging_struct = new_pml4_paddr;
-		s_kernel->load();
-
-		free_kmalloc_paging_structure(old_pdpt, 2);
-		kfree(reinterpret_cast<void*>(kmalloc_vaddr_of(old_pdpt_paddr).value()));
-		kfree(reinterpret_cast<void*>(kmalloc_vaddr_of(old_pml4_paddr).value()));
-
-		allocate_zeroed_page_aligned_page = &FuncsHHDM::allocate_zeroed_page_aligned_page;
-		unallocate_page                   = &FuncsHHDM::unallocate_page;
-		V2P                               = &FuncsHHDM::V2P;
-		P2V                               = &FuncsHHDM::P2V;
-
-		s_is_post_heap_done = true;
-
-		// This is a hack to unmap fast page. fast page pt is copied
-		// while it is mapped, so we need to manually unmap it
-		SpinLockGuard _(s_fast_page_lock);
-		unmap_fast_page();
-	}
-
-	void PageTable::initial_load()
 	{
 		if (s_has_nxe)
 		{
@@ -423,8 +256,63 @@ namespace Kernel
 			"movq %%rax, %%cr0;"
 			::: "rax"
 		);
+	}
 
-		load();
+	void PageTable::initialize_and_load()
+	{
+		detect_cpu_features();
+		enable_cpu_features();
+
+		const paddr_t boot_pml4_paddr = ({
+			paddr_t paddr;
+			asm volatile("movq %%cr3, %0" : "=r"(paddr));
+			paddr;
+		});
+
+		initialize_hhdm(boot_pml4_paddr);
+
+		ASSERT(s_kernel == nullptr);
+		s_kernel = new PageTable();
+		ASSERT(s_kernel != nullptr);
+
+		s_kernel->m_highest_paging_struct = allocate_zeroed_page_aligned_page();
+		ASSERT(s_kernel->m_highest_paging_struct);
+
+		uint64_t* pml4 = P2V(s_kernel->m_highest_paging_struct);
+		memcpy(pml4, s_global_pml4_entries, sizeof(s_global_pml4_entries));
+		s_kernel->map_kernel_memory();
+		s_global_pml4_entries[511] = pml4[511];
+
+		// update fast page pt
+		{
+			constexpr vaddr_t uc_vaddr = uncanonicalize(fast_page());
+			constexpr uint16_t pml4e = (uc_vaddr >> 39) & 0x1FF;
+			constexpr uint16_t pdpte = (uc_vaddr >> 30) & 0x1FF;
+			constexpr uint16_t pde   = (uc_vaddr >> 21) & 0x1FF;
+
+			const auto get_or_allocate_entry =
+				[](paddr_t table_paddr, uint16_t entry, uint64_t flags)
+				{
+					uint64_t* table = P2V(table_paddr);
+
+					if (!(table[entry] & Flags::Present))
+					{
+						table[entry] = allocate_zeroed_page_aligned_page();
+						ASSERT(table[entry]);
+					}
+
+					table[entry] |= flags;
+
+					return table[entry] & s_page_addr_mask;
+				};
+
+			const paddr_t pml4 = s_kernel->m_highest_paging_struct;
+			const paddr_t pdpt = get_or_allocate_entry(pml4, pml4e, Flags::ReadWrite | Flags::Present);
+			const paddr_t pd   = get_or_allocate_entry(pdpt, pdpte, Flags::ReadWrite | Flags::Present);
+			s_fast_page_pt = P2V(get_or_allocate_entry(pd,   pde,   Flags::ReadWrite | Flags::Present));
+		}
+
+		s_kernel->load();
 	}
 
 	PageTable& PageTable::kernel()
@@ -440,12 +328,12 @@ namespace Kernel
 		return true;
 	}
 
-	void PageTable::initialize_kernel()
+	void PageTable::map_kernel_memory()
 	{
 		// Map (phys_kernel_start -> phys_kernel_end) to (virt_kernel_start -> virt_kernel_end)
 		const vaddr_t kernel_start = reinterpret_cast<vaddr_t>(g_kernel_start);
 		map_range_at(
-			V2P(kernel_start),
+			kernel_start - KERNEL_OFFSET,
 			kernel_start,
 			g_kernel_end - g_kernel_start,
 			Flags::Present
@@ -454,7 +342,7 @@ namespace Kernel
 		// Map executable kernel memory as executable
 		const vaddr_t kernel_execute_start = reinterpret_cast<vaddr_t>(g_kernel_execute_start);
 		map_range_at(
-			V2P(kernel_execute_start),
+			kernel_execute_start - KERNEL_OFFSET,
 			kernel_execute_start,
 			g_kernel_execute_end - g_kernel_execute_start,
 			Flags::Execute | Flags::Present
@@ -463,7 +351,7 @@ namespace Kernel
 		// Map writable kernel memory as writable
 		const vaddr_t kernel_writable_start = reinterpret_cast<vaddr_t>(g_kernel_writable_start);
 		map_range_at(
-			V2P(kernel_writable_start),
+			kernel_writable_start - KERNEL_OFFSET,
 			kernel_writable_start,
 			g_kernel_writable_end - g_kernel_writable_start,
 			Flags::ReadWrite | Flags::Present
@@ -472,114 +360,58 @@ namespace Kernel
 		// Map userspace memory
 		const vaddr_t userspace_start = reinterpret_cast<vaddr_t>(g_userspace_start);
 		map_range_at(
-			V2P(userspace_start),
+			userspace_start - KERNEL_OFFSET,
 			userspace_start,
 			g_userspace_end - g_userspace_start,
 			Flags::Execute | Flags::UserSupervisor | Flags::Present
 		);
 	}
 
-	void PageTable::prepare_fast_page()
-	{
-		constexpr vaddr_t uc_vaddr = uncanonicalize(fast_page());
-		constexpr uint64_t pml4e = (uc_vaddr >> 39) & 0x1FF;
-		constexpr uint64_t pdpte = (uc_vaddr >> 30) & 0x1FF;
-		constexpr uint64_t pde   = (uc_vaddr >> 21) & 0x1FF;
-		constexpr uint64_t pte   = (uc_vaddr >> 12) & 0x1FF;
-
-		uint64_t* pml4 = P2V(m_highest_paging_struct);
-		ASSERT(!(pml4[pml4e] & Flags::Present));
-		pml4[pml4e] = allocate_zeroed_page_aligned_page() | Flags::ReadWrite | Flags::Present;
-
-		uint64_t* pdpt = P2V(pml4[pml4e] & s_page_addr_mask);
-		ASSERT(!(pdpt[pdpte] & Flags::Present));
-		pdpt[pdpte] = allocate_zeroed_page_aligned_page() | Flags::ReadWrite | Flags::Present;
-
-		uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
-		ASSERT(!(pd[pde] & Flags::Present));
-		pd[pde] = allocate_zeroed_page_aligned_page() | Flags::ReadWrite | Flags::Present;
-
-		uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
-		ASSERT(pt[pte] == 0);
-		pt[pte] = Flags::Reserved;
-	}
-
 	void PageTable::map_fast_page(paddr_t paddr)
 	{
-		ASSERT(s_kernel);
+		ASSERT(paddr && paddr % PAGE_SIZE == 0);
-		ASSERT(paddr);
-		ASSERT(paddr % PAGE_SIZE == 0);
 
+		ASSERT(s_fast_page_pt);
 		ASSERT(s_fast_page_lock.current_processor_has_lock());
 
-		constexpr vaddr_t uc_vaddr = uncanonicalize(fast_page());
+		ASSERT(!(*s_fast_page_pt & Flags::Present));
-		constexpr uint64_t pml4e = (uc_vaddr >> 39) & 0x1FF;
+		s_fast_page_pt[0] = paddr | Flags::ReadWrite | Flags::Present;
-		constexpr uint64_t pdpte = (uc_vaddr >> 30) & 0x1FF;
-		constexpr uint64_t pde   = (uc_vaddr >> 21) & 0x1FF;
-		constexpr uint64_t pte   = (uc_vaddr >> 12) & 0x1FF;
 
-		const uint64_t* pml4 = P2V(s_kernel->m_highest_paging_struct);
+		asm volatile("invlpg (%0)" :: "r"(fast_page()));
-		const uint64_t* pdpt = P2V(pml4[pml4e] & s_page_addr_mask);
-		const uint64_t* pd   = P2V(pdpt[pdpte] & s_page_addr_mask);
-		      uint64_t* pt   = P2V(pd[pde] & s_page_addr_mask);
-
-		ASSERT(!(pt[pte] & Flags::Present));
-		pt[pte] = paddr | Flags::ReadWrite | Flags::Present;
-
-		asm volatile("invlpg (%0)" :: "r"(fast_page()) : "memory");
 	}
 
 	void PageTable::unmap_fast_page()
 	{
-		ASSERT(s_kernel);
+		ASSERT(s_fast_page_pt);
-
 		ASSERT(s_fast_page_lock.current_processor_has_lock());
 
-		constexpr vaddr_t uc_vaddr = uncanonicalize(fast_page());
+		ASSERT((*s_fast_page_pt & Flags::Present));
-		constexpr uint64_t pml4e = (uc_vaddr >> 39) & 0x1FF;
+		s_fast_page_pt[0] = 0;
-		constexpr uint64_t pdpte = (uc_vaddr >> 30) & 0x1FF;
-		constexpr uint64_t pde   = (uc_vaddr >> 21) & 0x1FF;
-		constexpr uint64_t pte   = (uc_vaddr >> 12) & 0x1FF;
 
-		const uint64_t* pml4 = P2V(s_kernel->m_highest_paging_struct);
+		asm volatile("invlpg (%0)" :: "r"(fast_page()));
-		const uint64_t* pdpt = P2V(pml4[pml4e] & s_page_addr_mask);
-		const uint64_t* pd   = P2V(pdpt[pdpte] & s_page_addr_mask);
-		      uint64_t* pt   = P2V(pd[pde] & s_page_addr_mask);
-
-		ASSERT(pt[pte] & Flags::Present);
-		pt[pte] = Flags::Reserved;
-
-		asm volatile("invlpg (%0)" :: "r"(fast_page()) : "memory");
 	}
 
 	BAN::ErrorOr<PageTable*> PageTable::create_userspace()
 	{
 		SpinLockGuard _(s_kernel->m_lock);
+
 		PageTable* page_table = new PageTable;
 		if (page_table == nullptr)
 			return BAN::Error::from_errno(ENOMEM);
-		page_table->map_kernel_memory();
+
+		page_table->m_highest_paging_struct = allocate_zeroed_page_aligned_page();
+		if (page_table->m_highest_paging_struct == 0)
+		{
+			delete page_table;
+			return BAN::Error::from_errno(ENOMEM);
+		}
+
+		uint64_t* pml4 = P2V(page_table->m_highest_paging_struct);
+		memcpy(pml4, s_global_pml4_entries, sizeof(s_global_pml4_entries));
+
 		return page_table;
 	}
 
-	void PageTable::map_kernel_memory()
-	{
-		ASSERT(s_kernel);
-		ASSERT(s_global_pml4_entries[511]);
-		ASSERT(m_highest_paging_struct == 0);
-		m_highest_paging_struct = allocate_zeroed_page_aligned_page();
-
-		PageTable::with_fast_page(m_highest_paging_struct, [] {
-			for (size_t i = 0; i < 512; i++)
-			{
-				if (s_global_pml4_entries[i] == 0)
-					continue;
-				ASSERT(i >= 256);
-				PageTable::fast_page_as_sized<uint64_t>(i) = s_global_pml4_entries[i];
-			}
-		});
-	}
-
 	PageTable::~PageTable()
 	{
 		if (m_highest_paging_struct == 0)
@@ -624,7 +456,7 @@ namespace Kernel
 		const bool is_userspace = (vaddr < KERNEL_OFFSET);
 		if (is_userspace && this != &PageTable::current())
 			;
-		else if (pages <= 32 || !s_is_post_heap_done)
+		else if (pages <= 32 || !s_is_initialized)
 		{
 			for (size_t i = 0; i < pages; i++, vaddr += PAGE_SIZE)
 				asm volatile("invlpg (%0)" :: "r"(vaddr));

kernel/arch/x86_64/boot.S

@@ -97,27 +97,25 @@ bananboot_end:
 .align 4096
 boot_pml4:
 	.quad V2P(boot_pdpt_lo) + (PG_READ_WRITE | PG_PRESENT)
-	.rept 510
+	.skip 510 * 8
-		.quad 0
-	.endr
 	.quad V2P(boot_pdpt_hi) + (PG_READ_WRITE | PG_PRESENT)
 boot_pdpt_lo:
 	.quad V2P(boot_pd) + (PG_READ_WRITE | PG_PRESENT)
-	.rept 511
+	.skip 511 * 8
-		.quad 0
-	.endr
 boot_pdpt_hi:
-	.rept 510
+	.skip 510 * 8
-		.quad 0
-	.endr
 	.quad V2P(boot_pd) + (PG_READ_WRITE | PG_PRESENT)
-	.quad 0
+	.skip 8
 boot_pd:
 	.set i, 0
-	.rept 512
+	.rept 511
 		.quad i + (PG_PAGE_SIZE | PG_READ_WRITE | PG_PRESENT)
 		.set i, i + 0x200000
 	.endr
+	.quad V2P(g_boot_fast_page_pt) + (PG_READ_WRITE | PG_PRESENT)
+.global g_boot_fast_page_pt
+g_boot_fast_page_pt:
+	.skip 512 * 8
 
 boot_gdt:
 	.quad 0x0000000000000000 # null descriptor
@@ -273,7 +271,7 @@ system_halt:
 	jmp 1b
 
 
-#define AP_V2P(vaddr) ((vaddr) - ap_trampoline + 0xF000)
+#define AP_REL(vaddr) ((vaddr) - ap_trampoline + 0xF000)
 
 .section .ap_init, "ax"
 
@@ -283,21 +281,27 @@ ap_trampoline:
 	jmp 1f
 
 .align 8
-ap_stack_ptr:
+ap_stack_paddr:
-	.skip 4
+	.skip 8
-ap_stack_loaded:
+ap_stack_vaddr:
-	.skip 1
+	.skip 8
+ap_prepare_paging:
+	.skip 8
+ap_page_table:
+	.skip 8
+ap_ready:
+	.skip 8
 
 1:	cli; cld
-	ljmpl $0x00, $AP_V2P(ap_cs_clear)
+	ljmpl $0x00, $AP_REL(ap_cs_clear)
 
 ap_cs_clear:
 	# load ap gdt and enter protected mode
-	lgdt AP_V2P(ap_gdtr)
+	lgdt AP_REL(ap_gdtr)
 	movl %cr0, %eax
 	orb $1, %al
 	movl %eax, %cr0
-	ljmpl $0x08, $AP_V2P(ap_protected_mode)
+	ljmpl $0x08, $AP_REL(ap_protected_mode)
 
 .code32
 ap_protected_mode:
@@ -306,8 +310,7 @@ ap_protected_mode:
 	movw %ax, %ss
 	movw %ax, %es
 
-	movl AP_V2P(ap_stack_ptr), %esp
+	movl AP_REL(ap_stack_paddr), %esp
-	movb $1, AP_V2P(ap_stack_loaded)
 
 	leal V2P(enable_sse),        %ecx; call *%ecx
 	leal V2P(enable_tsc),        %ecx; call *%ecx
@@ -315,28 +318,34 @@ ap_protected_mode:
 
 	# load boot gdt and enter long mode
 	lgdt V2P(boot_gdtr)
-	ljmpl $0x08, $AP_V2P(ap_long_mode)
+	ljmpl $0x08, $AP_REL(ap_long_mode)
 
 .code64
 ap_long_mode:
-	# move stack pointer to higher half
+	movq $ap_higher_half, %rax
-	movl %esp, %esp
+	jmp *%rax
-	addq $KERNEL_OFFSET, %rsp
+
+ap_higher_half:
+	movq AP_REL(ap_prepare_paging), %rax
+	call *%rax
+
+	# load AP's initial values
+	movq AP_REL(ap_stack_vaddr), %rsp
+	movq AP_REL(ap_page_table), %rax
+	movq $1, AP_REL(ap_ready)
+	movq %rax, %cr3
 
 	# clear rbp for stacktrace
 	xorq %rbp, %rbp
 
-	xorb %al, %al
 1:	pause
-	cmpb %al, g_ap_startup_done
+	cmpb $0, g_ap_startup_done
-	jz 1b
+	je 1b
 
 	lock incb g_ap_running_count
 
-	# jump to ap_main in higher half
+	call ap_main
-	movabsq $ap_main, %rcx
+	jmp system_halt
-	call *%rcx
-	jmp V2P(system_halt)
 
 ap_gdt:
 	.quad 0x0000000000000000 # null descriptor

kernel/arch/x86_64/linker.ld

@@ -15,17 +15,17 @@ SECTIONS
 		*(.bananboot)
 		*(.text.*)
 	}
+	.ap_init ALIGN(4K) : AT(ADDR(.ap_init) - KERNEL_OFFSET)
+	{
+		g_ap_init_addr = .;
+		*(.ap_init)
+		g_kernel_execute_end = .;
+	}
 	.userspace ALIGN(4K) : AT(ADDR(.userspace) - KERNEL_OFFSET)
 	{
 		g_userspace_start = .;
 		*(.userspace)
 		g_userspace_end = .;
-		g_kernel_execute_end = .;
-	}
-	.ap_init ALIGN(4K) : AT(ADDR(.ap_init) - KERNEL_OFFSET)
-	{
-		g_ap_init_addr = .;
-		*(.ap_init)
 	}
 	.rodata ALIGN(4K) : AT(ADDR(.rodata) - KERNEL_OFFSET)
 	{

kernel/include/kernel/Memory/PageTable.h

@@ -46,13 +46,22 @@ namespace Kernel
 		};
 
 	public:
-		static void initialize_pre_heap();
+		static void initialize_fast_page();
-		static void initialize_post_heap();
+		static void initialize_and_load();
+
+		static void enable_cpu_features();
 
 		static PageTable& kernel();
 		static PageTable& current() { return *reinterpret_cast<PageTable*>(Processor::get_current_page_table()); }
 
-		static constexpr vaddr_t fast_page() { return KERNEL_OFFSET; }
+		static constexpr vaddr_t fast_page()
+		{
+#if ARCH(x86_64)
+			return 0xffffffffbfe00000;
+#elif ARCH(i686)
+			return 0xffe00000;
+#endif
+		}
 
 		template<with_fast_page_callback F>
 		static void with_fast_page(paddr_t paddr, F callback)
@@ -121,7 +130,6 @@ namespace Kernel
 		vaddr_t reserve_free_contiguous_pages(size_t page_count, vaddr_t first_address, vaddr_t last_address = UINTPTR_MAX);
 
 		void load();
-		void initial_load();
 
 		void invalidate_page(vaddr_t addr, bool send_smp_message) { invalidate_range(addr, 1, send_smp_message); }
 		void invalidate_range(vaddr_t addr, size_t pages, bool send_smp_message);
@@ -129,14 +137,14 @@ namespace Kernel
 		InterruptState lock() const { return m_lock.lock(); }
 		void unlock(InterruptState state) const { m_lock.unlock(state); }
 
+		paddr_t paddr() const { return m_highest_paging_struct; }
+
 		void debug_dump();
 
 	private:
 		PageTable() = default;
 		uint64_t get_page_data(vaddr_t) const;
-		void initialize_kernel();
 		void map_kernel_memory();
-		void prepare_fast_page();
 
 		static void map_fast_page(paddr_t);
 		static void unmap_fast_page();

kernel/include/kernel/Processor.h

@@ -58,6 +58,8 @@ namespace Kernel
 		static Processor& create(ProcessorID id);
 		static Processor& initialize();
 
+		void allocate_stack();
+
 		static ProcessorID current_id() { return read_gs_sized<ProcessorID>(offsetof(Processor, m_id)); }
 		static uint8_t current_index()  { return read_gs_sized<uint8_t>(offsetof(Processor, m_index)); }
 		static ProcessorID id_from_index(size_t index);
@@ -100,11 +102,8 @@ namespace Kernel
 				handle_smp_messages();
 		}
 
-		static uintptr_t current_stack_bottom() { return read_gs_sized<uintptr_t>(offsetof(Processor, m_stack)); }
+		vaddr_t stack_top_vaddr() const { return m_stack_vaddr + s_stack_size; }
-		static uintptr_t current_stack_top()	{ return current_stack_bottom() + s_stack_size; }
+		paddr_t stack_top_paddr() const { return m_stack_paddr + s_stack_size; }
-
-		uintptr_t stack_bottom() const	{ return reinterpret_cast<uintptr_t>(m_stack); }
-		uintptr_t stack_top() const		{ return stack_bottom() + s_stack_size; }
 
 		static void set_thread_syscall_stack(vaddr_t vaddr) { write_gs_sized<vaddr_t>(offsetof(Processor, m_thread_syscall_stack), vaddr); }
 
@@ -215,8 +214,9 @@ namespace Kernel
 
 		Thread* m_sse_thread { nullptr };
 
-		static constexpr size_t s_stack_size { 4096 };
+		static constexpr size_t s_stack_size { PAGE_SIZE };
-		void* m_stack { nullptr };
+		vaddr_t m_stack_vaddr { 0 };
+		paddr_t m_stack_paddr { 0 };
 
 		GDT* m_gdt { nullptr };
 		IDT* m_idt { nullptr };

kernel/kernel/APIC.cpp

@@ -293,9 +293,25 @@ namespace Kernel
 			dprintln("Trying to enable processor (lapic id {})", processor.apic_id);
 
 			auto& proc = Kernel::Processor::create(ProcessorID(processor.apic_id));
+			proc.allocate_stack();
+
+			struct ap_init_info_t
+			{
+				uintptr_t stack_paddr;
+				uintptr_t stack_vaddr;
+				uintptr_t prepare_paging;
+				uintptr_t page_table;
+				uintptr_t ready;
+			};
+
 			PageTable::with_fast_page(ap_init_paddr, [&] {
-				PageTable::fast_page_as_sized<uint32_t>(2) = kmalloc_paddr_of(proc.stack_top()).value();
+				PageTable::fast_page_as<ap_init_info_t>(8) = {
-				PageTable::fast_page_as_sized<uint8_t>(13) = 0;
+					.stack_paddr    = static_cast<uintptr_t>(proc.stack_top_paddr()),
+					.stack_vaddr    = proc.stack_top_vaddr(),
+					.prepare_paging = reinterpret_cast<uintptr_t>(&PageTable::enable_cpu_features),
+					.page_table     = static_cast<uintptr_t>(PageTable::kernel().paddr()),
+					.ready          = 0,
+				};
 			});
 
 			write_to_local_apic(LAPIC_ERROR_REG, 0x00);
@@ -334,12 +350,9 @@ namespace Kernel
 
 			// give processor upto 100 * 100 us + 200 us to boot
 			PageTable::with_fast_page(ap_init_paddr, [&] {
-				for (int i = 0; i < 100; i++)
+				for (int i = 0; i < 100; i++, udelay(100))
-				{
+					if (__atomic_load_n(&PageTable::fast_page_as<ap_init_info_t>(8).ready, __ATOMIC_SEQ_CST))
-					if (__atomic_load_n(&PageTable::fast_page_as_sized<uint8_t>(13), __ATOMIC_SEQ_CST))
 						break;
-					udelay(100);
-				}
 			});
 
 			initialized_aps++;

kernel/kernel/Processor.cpp

@@ -73,9 +73,6 @@ namespace Kernel
 		ASSERT(processor.m_id == PROCESSOR_NONE);
 		processor.m_id = id;
 
-		processor.m_stack = kmalloc(s_stack_size, 4096, true);
-		ASSERT(processor.m_stack);
-
 		processor.m_gdt = GDT::create(&processor);
 		ASSERT(processor.m_gdt);
 
@@ -157,6 +154,21 @@ namespace Kernel
 		return processor;
 	}
 
+	// NOTE: I don't like this being a separate function but we need heap and page tables for this :)
+	void Processor::allocate_stack()
+	{
+		ASSERT(m_stack_paddr == 0);
+		ASSERT(m_stack_vaddr == 0);
+
+		m_stack_paddr = Heap::get().take_free_page();
+		ASSERT(m_stack_paddr);
+
+		m_stack_vaddr = PageTable::kernel().reserve_free_page(KERNEL_OFFSET);
+		ASSERT(m_stack_vaddr);
+
+		PageTable::kernel().map_page_at(m_stack_paddr, m_stack_vaddr, PageTable::ReadWrite | PageTable::Present);
+	}
+
 	void Processor::initialize_smp()
 	{
 		const auto processor_id = current_id();
@@ -565,7 +577,7 @@ namespace Kernel
 		if (!scheduler().is_idle())
 			Thread::current().set_cpu_time_stop();
 
-		asm_yield_trampoline(Processor::current_stack_top());
+		asm_yield_trampoline(processor_info.stack_top_vaddr());
 
 		processor_info.m_start_ns = SystemTimer::get().ns_since_boot();
 

kernel/kernel/kernel.cpp

@@ -126,19 +126,20 @@ extern "C" void kernel_main(uint32_t boot_magic, uint32_t boot_info)
 	parse_boot_info(boot_magic, boot_info);
 	dprintln("boot info parsed");
 
-	Processor::create(PROCESSOR_NONE);
+	auto& processor = Processor::create(PROCESSOR_NONE);
 	Processor::initialize();
 	dprintln("BSP initialized");
 
-	PageTable::initialize_pre_heap();
+	PageTable::initialize_fast_page();
-	PageTable::kernel().initial_load();
+	dprintln("fast page initialized");
-	dprintln("PageTable stage1 initialized");
 
 	Heap::initialize();
-	dprintln("Heap initialzed");
+	dprintln("Heap initialized");
 
-	PageTable::initialize_post_heap();
+	PageTable::initialize_and_load();
-	dprintln("PageTable stage2 initialized");
+	dprintln("PageTable initialized");
+
+	processor.allocate_stack();
 
 	parse_command_line();
 	dprintln("command line parsed, root='{}', console='{}'", cmdline.root, cmdline.console);
@@ -270,7 +271,7 @@ extern "C" void ap_main()
 	using namespace Kernel;
 
 	Processor::initialize();
-	PageTable::kernel().initial_load();
+
 	InterruptController::get().enable();
 
 	Processor::wait_until_processors_ready();