diff --git a/kernel/arch/i686/PageTable.cpp b/kernel/arch/i686/PageTable.cpp index 285024df..4246e64f 100644 --- a/kernel/arch/i686/PageTable.cpp +++ b/kernel/arch/i686/PageTable.cpp @@ -16,6 +16,8 @@ extern uint8_t g_kernel_writable_end[]; extern uint8_t g_userspace_start[]; extern uint8_t g_userspace_end[]; +extern uint64_t g_boot_fast_page_pt[]; + namespace Kernel { @@ -24,7 +26,7 @@ namespace Kernel constexpr uint64_t s_page_flag_mask = 0x8000000000000FFF; constexpr uint64_t s_page_addr_mask = ~s_page_flag_mask; - static bool s_is_post_heap_done = false; + static bool s_is_initialized = false; static PageTable* s_kernel = nullptr; static bool s_has_nxe = false; @@ -33,6 +35,28 @@ namespace Kernel static paddr_t s_global_pdpte = 0; + static uint64_t* s_fast_page_pt { nullptr }; + + static uint64_t* allocate_zeroed_page_aligned_page() + { + void* page = kmalloc(PAGE_SIZE, PAGE_SIZE, true); + ASSERT(page); + memset(page, 0, PAGE_SIZE); + return (uint64_t*)page; + } + + template + static paddr_t V2P(const T vaddr) + { + return (vaddr_t)vaddr - KERNEL_OFFSET + g_boot_info.kernel_paddr; + } + + template + static uint64_t* P2V(const T paddr) + { + return reinterpret_cast(reinterpret_cast(paddr) - g_boot_info.kernel_paddr + KERNEL_OFFSET); + } + static inline PageTable::flags_t parse_flags(uint64_t entry) { using Flags = PageTable::Flags; @@ -51,31 +75,22 @@ namespace Kernel return result; } - void PageTable::initialize_pre_heap() + void PageTable::initialize_fast_page() + { + s_fast_page_pt = g_boot_fast_page_pt; + } + + static void detect_cpu_features() { if (CPUID::has_nxe()) s_has_nxe = true; - if (CPUID::has_pge()) s_has_pge = true; - if (CPUID::has_pat()) s_has_pat = true; - - ASSERT(s_kernel == nullptr); - s_kernel = new PageTable(); - ASSERT(s_kernel); - - s_kernel->initialize_kernel(); - s_kernel->initial_load(); } - void PageTable::initialize_post_heap() - { - s_is_post_heap_done = true; - } - - void PageTable::initial_load() + void PageTable::enable_cpu_features() { if (s_has_nxe) { @@ -116,8 +131,56 @@ namespace Kernel "movl %%eax, %%cr0;" ::: "rax" ); + } - load(); + void PageTable::initialize_and_load() + { + detect_cpu_features(); + enable_cpu_features(); + + ASSERT(s_kernel == nullptr); + s_kernel = new PageTable(); + ASSERT(s_kernel); + + auto* pdpt = allocate_zeroed_page_aligned_page(); + ASSERT(pdpt); + + s_kernel->m_highest_paging_struct = V2P(pdpt); + s_kernel->map_kernel_memory(); + + PageTable::with_fast_page(s_kernel->m_highest_paging_struct, [] { + s_global_pdpte = PageTable::fast_page_as_sized(3); + }); + + // update fast page pt + { + constexpr vaddr_t vaddr = fast_page(); + constexpr uint16_t pdpte = (vaddr >> 30) & 0x1FF; + constexpr uint16_t pde = (vaddr >> 21) & 0x1FF; + + const auto get_or_allocate_entry = + [](paddr_t table_paddr, uint16_t entry, uint64_t flags) + { + uint64_t* table = P2V(table_paddr); + + if (!(table[entry] & Flags::Present)) + { + auto* vaddr = allocate_zeroed_page_aligned_page(); + ASSERT(vaddr); + table[entry] = V2P(vaddr); + } + + table[entry] |= flags; + + return table[entry] & s_page_addr_mask; + }; + + const paddr_t pdpt = s_kernel->m_highest_paging_struct; + const paddr_t pd = get_or_allocate_entry(pdpt, pdpte, Flags::Present); + s_fast_page_pt = P2V(get_or_allocate_entry(pd, pde, Flags::ReadWrite | Flags::Present)); + } + + s_kernel->load(); } PageTable& PageTable::kernel() @@ -131,40 +194,12 @@ namespace Kernel return true; } - static uint64_t* allocate_zeroed_page_aligned_page() + void PageTable::map_kernel_memory() { - void* page = kmalloc(PAGE_SIZE, PAGE_SIZE, true); - ASSERT(page); - memset(page, 0, PAGE_SIZE); - return (uint64_t*)page; - } - - template - static paddr_t V2P(const T vaddr) - { - return (vaddr_t)vaddr - KERNEL_OFFSET + g_boot_info.kernel_paddr; - } - - template - static uint64_t* P2V(const T paddr) - { - return reinterpret_cast(reinterpret_cast(paddr) - g_boot_info.kernel_paddr + KERNEL_OFFSET); - } - - void PageTable::initialize_kernel() - { - ASSERT(s_global_pdpte == 0); - s_global_pdpte = V2P(allocate_zeroed_page_aligned_page()); - - map_kernel_memory(); - - prepare_fast_page(); - // Map (phys_kernel_start -> phys_kernel_end) to (virt_kernel_start -> virt_kernel_end) - ASSERT((vaddr_t)g_kernel_start % PAGE_SIZE == 0); map_range_at( V2P(g_kernel_start), - (vaddr_t)g_kernel_start, + reinterpret_cast(g_kernel_start), g_kernel_end - g_kernel_start, Flags::Present ); @@ -172,7 +207,7 @@ namespace Kernel // Map executable kernel memory as executable map_range_at( V2P(g_kernel_execute_start), - (vaddr_t)g_kernel_execute_start, + reinterpret_cast(g_kernel_execute_start), g_kernel_execute_end - g_kernel_execute_start, Flags::Execute | Flags::Present ); @@ -180,7 +215,7 @@ namespace Kernel // Map writable kernel memory as writable map_range_at( V2P(g_kernel_writable_start), - (vaddr_t)g_kernel_writable_start, + reinterpret_cast(g_kernel_writable_start), g_kernel_writable_end - g_kernel_writable_start, Flags::ReadWrite | Flags::Present ); @@ -188,70 +223,34 @@ namespace Kernel // Map userspace memory map_range_at( V2P(g_userspace_start), - (vaddr_t)g_userspace_start, + reinterpret_cast(g_userspace_start), g_userspace_end - g_userspace_start, Flags::Execute | Flags::UserSupervisor | Flags::Present ); } - void PageTable::prepare_fast_page() - { - constexpr uint64_t pdpte = (fast_page() >> 30) & 0x1FF; - constexpr uint64_t pde = (fast_page() >> 21) & 0x1FF; - constexpr uint64_t pte = (fast_page() >> 12) & 0x1FF; - - const uint64_t* pdpt = P2V(m_highest_paging_struct); - ASSERT(pdpt[pdpte] & Flags::Present); - - uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask); - ASSERT(!(pd[pde] & Flags::Present)); - pd[pde] = V2P(allocate_zeroed_page_aligned_page()) | Flags::ReadWrite | Flags::Present; - - uint64_t* pt = P2V(pd[pde] & s_page_addr_mask); - ASSERT(pt[pte] == 0); - pt[pte] = Flags::Reserved; - } - void PageTable::map_fast_page(paddr_t paddr) { - ASSERT(s_kernel); - ASSERT(paddr); - ASSERT(paddr % PAGE_SIZE == 0); + ASSERT(paddr && paddr % PAGE_SIZE == 0); + ASSERT(s_fast_page_pt); ASSERT(s_fast_page_lock.current_processor_has_lock()); - constexpr uint64_t pdpte = (fast_page() >> 30) & 0x1FF; - constexpr uint64_t pde = (fast_page() >> 21) & 0x1FF; - constexpr uint64_t pte = (fast_page() >> 12) & 0x1FF; + ASSERT(!(*s_fast_page_pt & Flags::Present)); + s_fast_page_pt[0] = paddr | Flags::ReadWrite | Flags::Present; - uint64_t* pdpt = P2V(s_kernel->m_highest_paging_struct); - uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask); - uint64_t* pt = P2V(pd[pde] & s_page_addr_mask); - - ASSERT(!(pt[pte] & Flags::Present)); - pt[pte] = paddr | Flags::ReadWrite | Flags::Present; - - asm volatile("invlpg (%0)" :: "r"(fast_page()) : "memory"); + asm volatile("invlpg (%0)" :: "r"(fast_page())); } void PageTable::unmap_fast_page() { - ASSERT(s_kernel); - + ASSERT(s_fast_page_pt); ASSERT(s_fast_page_lock.current_processor_has_lock()); - constexpr uint64_t pdpte = (fast_page() >> 30) & 0x1FF; - constexpr uint64_t pde = (fast_page() >> 21) & 0x1FF; - constexpr uint64_t pte = (fast_page() >> 12) & 0x1FF; + ASSERT((*s_fast_page_pt & Flags::Present)); + s_fast_page_pt[0] = 0; - uint64_t* pdpt = P2V(s_kernel->m_highest_paging_struct); - uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask); - uint64_t* pt = P2V(pd[pde] & s_page_addr_mask); - - ASSERT(pt[pte] & Flags::Present); - pt[pte] = Flags::Reserved; - - asm volatile("invlpg (%0)" :: "r"(fast_page()) : "memory"); + asm volatile("invlpg (%0)" :: "r"(fast_page())); } BAN::ErrorOr PageTable::create_userspace() @@ -260,25 +259,23 @@ namespace Kernel PageTable* page_table = new PageTable; if (page_table == nullptr) return BAN::Error::from_errno(ENOMEM); - page_table->map_kernel_memory(); - return page_table; - } - void PageTable::map_kernel_memory() - { - ASSERT(s_kernel); - ASSERT(s_global_pdpte); + uint64_t* pdpt = allocate_zeroed_page_aligned_page(); + if (pdpt == nullptr) + { + delete page_table; + return BAN::Error::from_errno(ENOMEM); + } - ASSERT(m_highest_paging_struct == 0); - m_highest_paging_struct = V2P(kmalloc(32, 32, true)); - ASSERT(m_highest_paging_struct); + page_table->m_highest_paging_struct = V2P(pdpt); - uint64_t* pdpt = P2V(m_highest_paging_struct); pdpt[0] = 0; pdpt[1] = 0; pdpt[2] = 0; pdpt[3] = s_global_pdpte | Flags::Present; static_assert(KERNEL_OFFSET == 0xC0000000); + + return page_table; } PageTable::~PageTable() @@ -318,7 +315,7 @@ namespace Kernel const bool is_userspace = (vaddr < KERNEL_OFFSET); if (is_userspace && this != &PageTable::current()) ; - else if (pages <= 32 || !s_is_post_heap_done) + else if (pages <= 32 || !s_is_initialized) { for (size_t i = 0; i < pages; i++, vaddr += PAGE_SIZE) asm volatile("invlpg (%0)" :: "r"(vaddr)); diff --git a/kernel/arch/i686/boot.S b/kernel/arch/i686/boot.S index 9bb11bce..040f8a0f 100644 --- a/kernel/arch/i686/boot.S +++ b/kernel/arch/i686/boot.S @@ -98,8 +98,7 @@ bananboot_end: boot_pdpt: .long V2P(boot_pd) + (PG_PRESENT) .long 0 - .quad 0 - .quad 0 + .skip 2 * 8 .long V2P(boot_pd) + (PG_PRESENT) .long 0 .align 4096 @@ -112,13 +111,16 @@ boot_pd: .endr boot_pts: .set i, 0 - .rept 512 + .rept 511 .rept 512 .long i + (PG_READ_WRITE | PG_PRESENT) .long 0 .set i, i + 0x1000 .endr .endr +.global g_boot_fast_page_pt +g_boot_fast_page_pt: + .skip 512 * 8 boot_gdt: .quad 0x0000000000000000 # null descriptor @@ -274,7 +276,7 @@ system_halt: jmp 1b -#define AP_V2P(vaddr) ((vaddr) - ap_trampoline + 0xF000) +#define AP_REL(vaddr) ((vaddr) - ap_trampoline + 0xF000) .section .ap_init, "ax" @@ -284,21 +286,27 @@ ap_trampoline: jmp 1f .align 8 -ap_stack_ptr: +ap_stack_paddr: + .skip 4 +ap_stack_vaddr: + .skip 4 +ap_prepare_paging: + .skip 4 +ap_page_table: + .skip 4 +ap_ready: .skip 4 -ap_stack_loaded: - .skip 1 1: cli; cld - ljmpl $0x00, $AP_V2P(ap_cs_clear) + ljmpl $0x00, $AP_REL(ap_cs_clear) ap_cs_clear: # load ap gdt and enter protected mode - lgdt AP_V2P(ap_gdtr) + lgdt AP_REL(ap_gdtr) movl %cr0, %eax orb $1, %al movl %eax, %cr0 - ljmpl $0x08, $AP_V2P(ap_protected_mode) + ljmpl $0x08, $AP_REL(ap_protected_mode) .code32 ap_protected_mode: @@ -307,8 +315,7 @@ ap_protected_mode: movw %ax, %ss movw %ax, %es - movl AP_V2P(ap_stack_ptr), %esp - movb $1, AP_V2P(ap_stack_loaded) + movl AP_REL(ap_stack_paddr), %esp leal V2P(enable_sse), %ecx; call *%ecx leal V2P(enable_tsc), %ecx; call *%ecx @@ -316,24 +323,28 @@ ap_protected_mode: # load boot gdt and enter long mode lgdt V2P(boot_gdtr) - ljmpl $0x08, $AP_V2P(ap_flush_gdt) + ljmpl $0x08, $AP_REL(ap_flush_gdt) ap_flush_gdt: - # move stack pointer to higher half - movl %esp, %esp - addl $KERNEL_OFFSET, %esp - - # jump to higher half - leal ap_higher_half, %ecx + movl $ap_higher_half, %ecx jmp *%ecx ap_higher_half: + movl AP_REL(ap_prepare_paging), %eax + call *%eax + + # load AP's initial values + movl AP_REL(ap_stack_vaddr), %esp + movl AP_REL(ap_page_table), %eax + movl $1, AP_REL(ap_ready) + movl %eax, %cr3 + # clear rbp for stacktrace xorl %ebp, %ebp 1: pause cmpb $0, g_ap_startup_done - jz 1b + je 1b lock incb g_ap_running_count diff --git a/kernel/arch/i686/linker.ld b/kernel/arch/i686/linker.ld index 33abb58e..1c46606e 100644 --- a/kernel/arch/i686/linker.ld +++ b/kernel/arch/i686/linker.ld @@ -15,17 +15,17 @@ SECTIONS *(.bananboot) *(.text.*) } + .ap_init ALIGN(4K) : AT(ADDR(.ap_init) - KERNEL_OFFSET) + { + g_ap_init_addr = .; + *(.ap_init) + g_kernel_execute_end = .; + } .userspace ALIGN(4K) : AT(ADDR(.userspace) - KERNEL_OFFSET) { g_userspace_start = .; *(.userspace) g_userspace_end = .; - g_kernel_execute_end = .; - } - .ap_init ALIGN(4K) : AT(ADDR(.ap_init) - KERNEL_OFFSET) - { - g_ap_init_addr = .; - *(.ap_init) } .rodata ALIGN(4K) : AT(ADDR(.rodata) - KERNEL_OFFSET) { diff --git a/kernel/arch/x86_64/PageTable.cpp b/kernel/arch/x86_64/PageTable.cpp index ec310fdc..20ac1775 100644 --- a/kernel/arch/x86_64/PageTable.cpp +++ b/kernel/arch/x86_64/PageTable.cpp @@ -2,7 +2,6 @@ #include #include #include -#include #include extern uint8_t g_kernel_start[]; @@ -17,13 +16,15 @@ extern uint8_t g_kernel_writable_end[]; extern uint8_t g_userspace_start[]; extern uint8_t g_userspace_end[]; +extern uint64_t g_boot_fast_page_pt[]; + namespace Kernel { SpinLock PageTable::s_fast_page_lock; static constexpr vaddr_t s_hhdm_offset = 0xFFFF800000000000; - static bool s_is_post_heap_done = false; + static bool s_is_initialized = false; constexpr uint64_t s_page_flag_mask = 0x8000000000000FFF; constexpr uint64_t s_page_addr_mask = ~s_page_flag_mask; @@ -35,6 +36,8 @@ namespace Kernel static paddr_t s_global_pml4_entries[512] { 0 }; + static uint64_t* s_fast_page_pt { nullptr }; + static constexpr inline bool is_canonical(uintptr_t addr) { constexpr uintptr_t mask = 0xFFFF800000000000; @@ -54,68 +57,27 @@ namespace Kernel return addr; } - struct FuncsKmalloc + static paddr_t allocate_zeroed_page_aligned_page() { - static paddr_t allocate_zeroed_page_aligned_page() - { - void* page = kmalloc(PAGE_SIZE, PAGE_SIZE, true); - ASSERT(page); - memset(page, 0, PAGE_SIZE); - return kmalloc_paddr_of(reinterpret_cast(page)).value(); - } + const paddr_t paddr = Heap::get().take_free_page(); + ASSERT(paddr); + memset(reinterpret_cast(paddr + s_hhdm_offset), 0, PAGE_SIZE); + return paddr; + } - static void unallocate_page(paddr_t paddr) - { - kfree(reinterpret_cast(kmalloc_vaddr_of(paddr).value())); - } - - static paddr_t V2P(vaddr_t vaddr) - { - return vaddr - KERNEL_OFFSET + g_boot_info.kernel_paddr; - } - - static uint64_t* P2V(paddr_t paddr) - { - return reinterpret_cast(paddr - g_boot_info.kernel_paddr + KERNEL_OFFSET); - } - }; - - struct FuncsHHDM + static void unallocate_page(paddr_t paddr) { - static paddr_t allocate_zeroed_page_aligned_page() - { - const paddr_t paddr = Heap::get().take_free_page(); - ASSERT(paddr); - memset(reinterpret_cast(paddr + s_hhdm_offset), 0, PAGE_SIZE); - return paddr; - } + Heap::get().release_page(paddr); + } - static void unallocate_page(paddr_t paddr) - { - Heap::get().release_page(paddr); - } + static uint64_t* P2V(paddr_t paddr) + { + ASSERT(paddr != 0); + ASSERT(!BAN::Math::will_addition_overflow(paddr, s_hhdm_offset)); + return reinterpret_cast(paddr + s_hhdm_offset); + } - static paddr_t V2P(vaddr_t vaddr) - { - ASSERT(vaddr >= s_hhdm_offset); - ASSERT(vaddr < KERNEL_OFFSET); - return vaddr - s_hhdm_offset; - } - - static uint64_t* P2V(paddr_t paddr) - { - ASSERT(paddr != 0); - ASSERT(!BAN::Math::will_addition_overflow(paddr, s_hhdm_offset)); - return reinterpret_cast(paddr + s_hhdm_offset); - } - }; - - static paddr_t (*allocate_zeroed_page_aligned_page)() = &FuncsKmalloc::allocate_zeroed_page_aligned_page; - static void (*unallocate_page)(paddr_t) = &FuncsKmalloc::unallocate_page; - static paddr_t (*V2P)(vaddr_t) = &FuncsKmalloc::V2P; - static uint64_t* (*P2V)(paddr_t) = &FuncsKmalloc::P2V; - - static inline PageTable::flags_t parse_flags(uint64_t entry) + static PageTable::flags_t parse_flags(uint64_t entry) { using Flags = PageTable::Flags; @@ -137,7 +99,7 @@ namespace Kernel // 0: 4 KiB // 1: 2 MiB // 2: 1 GiB - static void init_map_hhdm_page(paddr_t pml4, paddr_t paddr, uint8_t page_size) + static void map_hhdm_page(paddr_t pml4, paddr_t paddr, uint8_t page_size) { ASSERT(0 <= page_size && page_size <= 2); @@ -184,7 +146,7 @@ namespace Kernel const uint64_t noexec_flag = s_has_nxe ? (static_cast(1) << 63) : 0; const paddr_t pdpt = get_or_allocate_entry(pml4, pml4e, noexec_flag); - s_global_pml4_entries[pml4e] = pdpt | hhdm_flags; + s_global_pml4_entries[pml4e] = pdpt | hhdm_flags | noexec_flag; paddr_t lowest_paddr = pdpt; uint16_t lowest_entry = pdpte; @@ -207,23 +169,11 @@ namespace Kernel }); } - static void init_map_hhdm(paddr_t pml4) + static void initialize_hhdm(paddr_t pml4) { for (const auto& entry : g_boot_info.memory_map_entries) { - bool should_map = false; - switch (entry.type) - { - case MemoryMapEntry::Type::Available: - should_map = true; - break; - case MemoryMapEntry::Type::ACPIReclaim: - case MemoryMapEntry::Type::ACPINVS: - case MemoryMapEntry::Type::Reserved: - should_map = false; - break; - } - if (!should_map) + if (entry.type != MemoryMapEntry::Type::Available) continue; constexpr size_t one_gib = 1024 * 1024 * 1024; @@ -235,156 +185,39 @@ namespace Kernel { if (s_has_gib && paddr % one_gib == 0 && paddr + one_gib <= entry_end) { - init_map_hhdm_page(pml4, paddr, 2); + map_hhdm_page(pml4, paddr, 2); paddr += one_gib; } else if (paddr % two_mib == 0 && paddr + two_mib <= entry_end) { - init_map_hhdm_page(pml4, paddr, 1); + map_hhdm_page(pml4, paddr, 1); paddr += two_mib; } else { - init_map_hhdm_page(pml4, paddr, 0); + map_hhdm_page(pml4, paddr, 0); paddr += PAGE_SIZE; } } } } - static paddr_t copy_page_from_kmalloc_to_heap(paddr_t kmalloc_paddr) + void PageTable::initialize_fast_page() { - const paddr_t heap_paddr = Heap::get().take_free_page(); - ASSERT(heap_paddr); - - const vaddr_t kmalloc_vaddr = kmalloc_vaddr_of(kmalloc_paddr).value(); - - PageTable::with_fast_page(heap_paddr, [kmalloc_vaddr] { - memcpy(PageTable::fast_page_as_ptr(), reinterpret_cast(kmalloc_vaddr), PAGE_SIZE); - }); - - return heap_paddr; + s_fast_page_pt = g_boot_fast_page_pt; } - static void copy_paging_structure_to_heap(uint64_t* old_table, uint64_t* new_table, int depth) - { - if (depth == 0) - return; - - constexpr uint64_t page_flag_mask = 0x8000000000000FFF; - constexpr uint64_t page_addr_mask = ~page_flag_mask; - - for (uint16_t index = 0; index < 512; index++) - { - const uint64_t old_entry = old_table[index]; - if (old_entry == 0) - { - new_table[index] = 0; - continue; - } - - const paddr_t old_paddr = old_entry & page_addr_mask; - const paddr_t new_paddr = copy_page_from_kmalloc_to_heap(old_paddr); - new_table[index] = new_paddr | (old_entry & page_flag_mask); - - uint64_t* next_old_table = reinterpret_cast(old_paddr + s_hhdm_offset); - uint64_t* next_new_table = reinterpret_cast(new_paddr + s_hhdm_offset); - copy_paging_structure_to_heap(next_old_table, next_new_table, depth - 1); - } - } - - static void free_kmalloc_paging_structure(uint64_t* table, int depth) - { - if (depth == 0) - return; - - constexpr uint64_t page_flag_mask = 0x8000000000000FFF; - constexpr uint64_t page_addr_mask = ~page_flag_mask; - - for (uint16_t index = 0; index < 512; index++) - { - const uint64_t entry = table[index]; - if (entry == 0) - continue; - - const paddr_t paddr = entry & page_addr_mask; - - uint64_t* next_table = reinterpret_cast(paddr + s_hhdm_offset); - free_kmalloc_paging_structure(next_table, depth - 1); - - kfree(reinterpret_cast(kmalloc_vaddr_of(paddr).value())); - } - } - - void PageTable::initialize_pre_heap() + static void detect_cpu_features() { if (CPUID::has_nxe()) s_has_nxe = true; - if (CPUID::has_pge()) s_has_pge = true; - if (CPUID::has_1gib_pages()) s_has_gib = true; - - ASSERT(s_kernel == nullptr); - s_kernel = new PageTable(); - ASSERT(s_kernel); - - s_kernel->m_highest_paging_struct = allocate_zeroed_page_aligned_page(); - s_kernel->prepare_fast_page(); - s_kernel->initialize_kernel(); - - for (auto pml4e : s_global_pml4_entries) - ASSERT(pml4e == 0); - const uint64_t* pml4 = P2V(s_kernel->m_highest_paging_struct); - s_global_pml4_entries[511] = pml4[511]; } - void PageTable::initialize_post_heap() - { - ASSERT(s_kernel); - - init_map_hhdm(s_kernel->m_highest_paging_struct); - - const paddr_t old_pml4_paddr = s_kernel->m_highest_paging_struct; - const paddr_t new_pml4_paddr = copy_page_from_kmalloc_to_heap(old_pml4_paddr); - - uint64_t* old_pml4 = reinterpret_cast(kmalloc_vaddr_of(old_pml4_paddr).value()); - uint64_t* new_pml4 = reinterpret_cast(new_pml4_paddr + s_hhdm_offset); - - const paddr_t old_pdpt_paddr = old_pml4[511] & s_page_addr_mask; - const paddr_t new_pdpt_paddr = Heap::get().take_free_page(); - ASSERT(new_pdpt_paddr); - - uint64_t* old_pdpt = reinterpret_cast(old_pdpt_paddr + s_hhdm_offset); - uint64_t* new_pdpt = reinterpret_cast(new_pdpt_paddr + s_hhdm_offset); - copy_paging_structure_to_heap(old_pdpt, new_pdpt, 2); - - new_pml4[511] = new_pdpt_paddr | (old_pml4[511] & s_page_flag_mask); - s_global_pml4_entries[511] = new_pml4[511]; - - s_kernel->m_highest_paging_struct = new_pml4_paddr; - s_kernel->load(); - - free_kmalloc_paging_structure(old_pdpt, 2); - kfree(reinterpret_cast(kmalloc_vaddr_of(old_pdpt_paddr).value())); - kfree(reinterpret_cast(kmalloc_vaddr_of(old_pml4_paddr).value())); - - allocate_zeroed_page_aligned_page = &FuncsHHDM::allocate_zeroed_page_aligned_page; - unallocate_page = &FuncsHHDM::unallocate_page; - V2P = &FuncsHHDM::V2P; - P2V = &FuncsHHDM::P2V; - - s_is_post_heap_done = true; - - // This is a hack to unmap fast page. fast page pt is copied - // while it is mapped, so we need to manually unmap it - SpinLockGuard _(s_fast_page_lock); - unmap_fast_page(); - } - - void PageTable::initial_load() + void PageTable::enable_cpu_features() { if (s_has_nxe) { @@ -423,8 +256,63 @@ namespace Kernel "movq %%rax, %%cr0;" ::: "rax" ); + } - load(); + void PageTable::initialize_and_load() + { + detect_cpu_features(); + enable_cpu_features(); + + const paddr_t boot_pml4_paddr = ({ + paddr_t paddr; + asm volatile("movq %%cr3, %0" : "=r"(paddr)); + paddr; + }); + + initialize_hhdm(boot_pml4_paddr); + + ASSERT(s_kernel == nullptr); + s_kernel = new PageTable(); + ASSERT(s_kernel != nullptr); + + s_kernel->m_highest_paging_struct = allocate_zeroed_page_aligned_page(); + ASSERT(s_kernel->m_highest_paging_struct); + + uint64_t* pml4 = P2V(s_kernel->m_highest_paging_struct); + memcpy(pml4, s_global_pml4_entries, sizeof(s_global_pml4_entries)); + s_kernel->map_kernel_memory(); + s_global_pml4_entries[511] = pml4[511]; + + // update fast page pt + { + constexpr vaddr_t uc_vaddr = uncanonicalize(fast_page()); + constexpr uint16_t pml4e = (uc_vaddr >> 39) & 0x1FF; + constexpr uint16_t pdpte = (uc_vaddr >> 30) & 0x1FF; + constexpr uint16_t pde = (uc_vaddr >> 21) & 0x1FF; + + const auto get_or_allocate_entry = + [](paddr_t table_paddr, uint16_t entry, uint64_t flags) + { + uint64_t* table = P2V(table_paddr); + + if (!(table[entry] & Flags::Present)) + { + table[entry] = allocate_zeroed_page_aligned_page(); + ASSERT(table[entry]); + } + + table[entry] |= flags; + + return table[entry] & s_page_addr_mask; + }; + + const paddr_t pml4 = s_kernel->m_highest_paging_struct; + const paddr_t pdpt = get_or_allocate_entry(pml4, pml4e, Flags::ReadWrite | Flags::Present); + const paddr_t pd = get_or_allocate_entry(pdpt, pdpte, Flags::ReadWrite | Flags::Present); + s_fast_page_pt = P2V(get_or_allocate_entry(pd, pde, Flags::ReadWrite | Flags::Present)); + } + + s_kernel->load(); } PageTable& PageTable::kernel() @@ -440,12 +328,12 @@ namespace Kernel return true; } - void PageTable::initialize_kernel() + void PageTable::map_kernel_memory() { // Map (phys_kernel_start -> phys_kernel_end) to (virt_kernel_start -> virt_kernel_end) const vaddr_t kernel_start = reinterpret_cast(g_kernel_start); map_range_at( - V2P(kernel_start), + kernel_start - KERNEL_OFFSET, kernel_start, g_kernel_end - g_kernel_start, Flags::Present @@ -454,7 +342,7 @@ namespace Kernel // Map executable kernel memory as executable const vaddr_t kernel_execute_start = reinterpret_cast(g_kernel_execute_start); map_range_at( - V2P(kernel_execute_start), + kernel_execute_start - KERNEL_OFFSET, kernel_execute_start, g_kernel_execute_end - g_kernel_execute_start, Flags::Execute | Flags::Present @@ -463,7 +351,7 @@ namespace Kernel // Map writable kernel memory as writable const vaddr_t kernel_writable_start = reinterpret_cast(g_kernel_writable_start); map_range_at( - V2P(kernel_writable_start), + kernel_writable_start - KERNEL_OFFSET, kernel_writable_start, g_kernel_writable_end - g_kernel_writable_start, Flags::ReadWrite | Flags::Present @@ -472,114 +360,58 @@ namespace Kernel // Map userspace memory const vaddr_t userspace_start = reinterpret_cast(g_userspace_start); map_range_at( - V2P(userspace_start), + userspace_start - KERNEL_OFFSET, userspace_start, g_userspace_end - g_userspace_start, Flags::Execute | Flags::UserSupervisor | Flags::Present ); } - void PageTable::prepare_fast_page() - { - constexpr vaddr_t uc_vaddr = uncanonicalize(fast_page()); - constexpr uint64_t pml4e = (uc_vaddr >> 39) & 0x1FF; - constexpr uint64_t pdpte = (uc_vaddr >> 30) & 0x1FF; - constexpr uint64_t pde = (uc_vaddr >> 21) & 0x1FF; - constexpr uint64_t pte = (uc_vaddr >> 12) & 0x1FF; - - uint64_t* pml4 = P2V(m_highest_paging_struct); - ASSERT(!(pml4[pml4e] & Flags::Present)); - pml4[pml4e] = allocate_zeroed_page_aligned_page() | Flags::ReadWrite | Flags::Present; - - uint64_t* pdpt = P2V(pml4[pml4e] & s_page_addr_mask); - ASSERT(!(pdpt[pdpte] & Flags::Present)); - pdpt[pdpte] = allocate_zeroed_page_aligned_page() | Flags::ReadWrite | Flags::Present; - - uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask); - ASSERT(!(pd[pde] & Flags::Present)); - pd[pde] = allocate_zeroed_page_aligned_page() | Flags::ReadWrite | Flags::Present; - - uint64_t* pt = P2V(pd[pde] & s_page_addr_mask); - ASSERT(pt[pte] == 0); - pt[pte] = Flags::Reserved; - } - void PageTable::map_fast_page(paddr_t paddr) { - ASSERT(s_kernel); - ASSERT(paddr); - ASSERT(paddr % PAGE_SIZE == 0); + ASSERT(paddr && paddr % PAGE_SIZE == 0); + ASSERT(s_fast_page_pt); ASSERT(s_fast_page_lock.current_processor_has_lock()); - constexpr vaddr_t uc_vaddr = uncanonicalize(fast_page()); - constexpr uint64_t pml4e = (uc_vaddr >> 39) & 0x1FF; - constexpr uint64_t pdpte = (uc_vaddr >> 30) & 0x1FF; - constexpr uint64_t pde = (uc_vaddr >> 21) & 0x1FF; - constexpr uint64_t pte = (uc_vaddr >> 12) & 0x1FF; + ASSERT(!(*s_fast_page_pt & Flags::Present)); + s_fast_page_pt[0] = paddr | Flags::ReadWrite | Flags::Present; - const uint64_t* pml4 = P2V(s_kernel->m_highest_paging_struct); - const uint64_t* pdpt = P2V(pml4[pml4e] & s_page_addr_mask); - const uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask); - uint64_t* pt = P2V(pd[pde] & s_page_addr_mask); - - ASSERT(!(pt[pte] & Flags::Present)); - pt[pte] = paddr | Flags::ReadWrite | Flags::Present; - - asm volatile("invlpg (%0)" :: "r"(fast_page()) : "memory"); + asm volatile("invlpg (%0)" :: "r"(fast_page())); } void PageTable::unmap_fast_page() { - ASSERT(s_kernel); - + ASSERT(s_fast_page_pt); ASSERT(s_fast_page_lock.current_processor_has_lock()); - constexpr vaddr_t uc_vaddr = uncanonicalize(fast_page()); - constexpr uint64_t pml4e = (uc_vaddr >> 39) & 0x1FF; - constexpr uint64_t pdpte = (uc_vaddr >> 30) & 0x1FF; - constexpr uint64_t pde = (uc_vaddr >> 21) & 0x1FF; - constexpr uint64_t pte = (uc_vaddr >> 12) & 0x1FF; + ASSERT((*s_fast_page_pt & Flags::Present)); + s_fast_page_pt[0] = 0; - const uint64_t* pml4 = P2V(s_kernel->m_highest_paging_struct); - const uint64_t* pdpt = P2V(pml4[pml4e] & s_page_addr_mask); - const uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask); - uint64_t* pt = P2V(pd[pde] & s_page_addr_mask); - - ASSERT(pt[pte] & Flags::Present); - pt[pte] = Flags::Reserved; - - asm volatile("invlpg (%0)" :: "r"(fast_page()) : "memory"); + asm volatile("invlpg (%0)" :: "r"(fast_page())); } BAN::ErrorOr PageTable::create_userspace() { SpinLockGuard _(s_kernel->m_lock); + PageTable* page_table = new PageTable; if (page_table == nullptr) return BAN::Error::from_errno(ENOMEM); - page_table->map_kernel_memory(); + + page_table->m_highest_paging_struct = allocate_zeroed_page_aligned_page(); + if (page_table->m_highest_paging_struct == 0) + { + delete page_table; + return BAN::Error::from_errno(ENOMEM); + } + + uint64_t* pml4 = P2V(page_table->m_highest_paging_struct); + memcpy(pml4, s_global_pml4_entries, sizeof(s_global_pml4_entries)); + return page_table; } - void PageTable::map_kernel_memory() - { - ASSERT(s_kernel); - ASSERT(s_global_pml4_entries[511]); - ASSERT(m_highest_paging_struct == 0); - m_highest_paging_struct = allocate_zeroed_page_aligned_page(); - - PageTable::with_fast_page(m_highest_paging_struct, [] { - for (size_t i = 0; i < 512; i++) - { - if (s_global_pml4_entries[i] == 0) - continue; - ASSERT(i >= 256); - PageTable::fast_page_as_sized(i) = s_global_pml4_entries[i]; - } - }); - } - PageTable::~PageTable() { if (m_highest_paging_struct == 0) @@ -624,7 +456,7 @@ namespace Kernel const bool is_userspace = (vaddr < KERNEL_OFFSET); if (is_userspace && this != &PageTable::current()) ; - else if (pages <= 32 || !s_is_post_heap_done) + else if (pages <= 32 || !s_is_initialized) { for (size_t i = 0; i < pages; i++, vaddr += PAGE_SIZE) asm volatile("invlpg (%0)" :: "r"(vaddr)); diff --git a/kernel/arch/x86_64/boot.S b/kernel/arch/x86_64/boot.S index 9403ac94..833011ba 100644 --- a/kernel/arch/x86_64/boot.S +++ b/kernel/arch/x86_64/boot.S @@ -97,27 +97,25 @@ bananboot_end: .align 4096 boot_pml4: .quad V2P(boot_pdpt_lo) + (PG_READ_WRITE | PG_PRESENT) - .rept 510 - .quad 0 - .endr + .skip 510 * 8 .quad V2P(boot_pdpt_hi) + (PG_READ_WRITE | PG_PRESENT) boot_pdpt_lo: .quad V2P(boot_pd) + (PG_READ_WRITE | PG_PRESENT) - .rept 511 - .quad 0 - .endr + .skip 511 * 8 boot_pdpt_hi: - .rept 510 - .quad 0 - .endr + .skip 510 * 8 .quad V2P(boot_pd) + (PG_READ_WRITE | PG_PRESENT) - .quad 0 + .skip 8 boot_pd: .set i, 0 - .rept 512 + .rept 511 .quad i + (PG_PAGE_SIZE | PG_READ_WRITE | PG_PRESENT) .set i, i + 0x200000 .endr + .quad V2P(g_boot_fast_page_pt) + (PG_READ_WRITE | PG_PRESENT) +.global g_boot_fast_page_pt +g_boot_fast_page_pt: + .skip 512 * 8 boot_gdt: .quad 0x0000000000000000 # null descriptor @@ -273,7 +271,7 @@ system_halt: jmp 1b -#define AP_V2P(vaddr) ((vaddr) - ap_trampoline + 0xF000) +#define AP_REL(vaddr) ((vaddr) - ap_trampoline + 0xF000) .section .ap_init, "ax" @@ -283,21 +281,27 @@ ap_trampoline: jmp 1f .align 8 -ap_stack_ptr: - .skip 4 -ap_stack_loaded: - .skip 1 +ap_stack_paddr: + .skip 8 +ap_stack_vaddr: + .skip 8 +ap_prepare_paging: + .skip 8 +ap_page_table: + .skip 8 +ap_ready: + .skip 8 1: cli; cld - ljmpl $0x00, $AP_V2P(ap_cs_clear) + ljmpl $0x00, $AP_REL(ap_cs_clear) ap_cs_clear: # load ap gdt and enter protected mode - lgdt AP_V2P(ap_gdtr) + lgdt AP_REL(ap_gdtr) movl %cr0, %eax orb $1, %al movl %eax, %cr0 - ljmpl $0x08, $AP_V2P(ap_protected_mode) + ljmpl $0x08, $AP_REL(ap_protected_mode) .code32 ap_protected_mode: @@ -306,8 +310,7 @@ ap_protected_mode: movw %ax, %ss movw %ax, %es - movl AP_V2P(ap_stack_ptr), %esp - movb $1, AP_V2P(ap_stack_loaded) + movl AP_REL(ap_stack_paddr), %esp leal V2P(enable_sse), %ecx; call *%ecx leal V2P(enable_tsc), %ecx; call *%ecx @@ -315,28 +318,34 @@ ap_protected_mode: # load boot gdt and enter long mode lgdt V2P(boot_gdtr) - ljmpl $0x08, $AP_V2P(ap_long_mode) + ljmpl $0x08, $AP_REL(ap_long_mode) .code64 ap_long_mode: - # move stack pointer to higher half - movl %esp, %esp - addq $KERNEL_OFFSET, %rsp + movq $ap_higher_half, %rax + jmp *%rax + +ap_higher_half: + movq AP_REL(ap_prepare_paging), %rax + call *%rax + + # load AP's initial values + movq AP_REL(ap_stack_vaddr), %rsp + movq AP_REL(ap_page_table), %rax + movq $1, AP_REL(ap_ready) + movq %rax, %cr3 # clear rbp for stacktrace xorq %rbp, %rbp - xorb %al, %al 1: pause - cmpb %al, g_ap_startup_done - jz 1b + cmpb $0, g_ap_startup_done + je 1b lock incb g_ap_running_count - # jump to ap_main in higher half - movabsq $ap_main, %rcx - call *%rcx - jmp V2P(system_halt) + call ap_main + jmp system_halt ap_gdt: .quad 0x0000000000000000 # null descriptor diff --git a/kernel/arch/x86_64/linker.ld b/kernel/arch/x86_64/linker.ld index ef80c843..43e9ce4d 100644 --- a/kernel/arch/x86_64/linker.ld +++ b/kernel/arch/x86_64/linker.ld @@ -15,17 +15,17 @@ SECTIONS *(.bananboot) *(.text.*) } + .ap_init ALIGN(4K) : AT(ADDR(.ap_init) - KERNEL_OFFSET) + { + g_ap_init_addr = .; + *(.ap_init) + g_kernel_execute_end = .; + } .userspace ALIGN(4K) : AT(ADDR(.userspace) - KERNEL_OFFSET) { g_userspace_start = .; *(.userspace) g_userspace_end = .; - g_kernel_execute_end = .; - } - .ap_init ALIGN(4K) : AT(ADDR(.ap_init) - KERNEL_OFFSET) - { - g_ap_init_addr = .; - *(.ap_init) } .rodata ALIGN(4K) : AT(ADDR(.rodata) - KERNEL_OFFSET) { diff --git a/kernel/include/kernel/Memory/PageTable.h b/kernel/include/kernel/Memory/PageTable.h index 53b07b6c..68629266 100644 --- a/kernel/include/kernel/Memory/PageTable.h +++ b/kernel/include/kernel/Memory/PageTable.h @@ -46,13 +46,22 @@ namespace Kernel }; public: - static void initialize_pre_heap(); - static void initialize_post_heap(); + static void initialize_fast_page(); + static void initialize_and_load(); + + static void enable_cpu_features(); static PageTable& kernel(); static PageTable& current() { return *reinterpret_cast(Processor::get_current_page_table()); } - static constexpr vaddr_t fast_page() { return KERNEL_OFFSET; } + static constexpr vaddr_t fast_page() + { +#if ARCH(x86_64) + return 0xffffffffbfe00000; +#elif ARCH(i686) + return 0xffe00000; +#endif + } template static void with_fast_page(paddr_t paddr, F callback) @@ -121,7 +130,6 @@ namespace Kernel vaddr_t reserve_free_contiguous_pages(size_t page_count, vaddr_t first_address, vaddr_t last_address = UINTPTR_MAX); void load(); - void initial_load(); void invalidate_page(vaddr_t addr, bool send_smp_message) { invalidate_range(addr, 1, send_smp_message); } void invalidate_range(vaddr_t addr, size_t pages, bool send_smp_message); @@ -129,14 +137,14 @@ namespace Kernel InterruptState lock() const { return m_lock.lock(); } void unlock(InterruptState state) const { m_lock.unlock(state); } + paddr_t paddr() const { return m_highest_paging_struct; } + void debug_dump(); private: PageTable() = default; uint64_t get_page_data(vaddr_t) const; - void initialize_kernel(); void map_kernel_memory(); - void prepare_fast_page(); static void map_fast_page(paddr_t); static void unmap_fast_page(); diff --git a/kernel/include/kernel/Processor.h b/kernel/include/kernel/Processor.h index 3b66cddf..f86fe580 100644 --- a/kernel/include/kernel/Processor.h +++ b/kernel/include/kernel/Processor.h @@ -58,6 +58,8 @@ namespace Kernel static Processor& create(ProcessorID id); static Processor& initialize(); + void allocate_stack(); + static ProcessorID current_id() { return read_gs_sized(offsetof(Processor, m_id)); } static uint8_t current_index() { return read_gs_sized(offsetof(Processor, m_index)); } static ProcessorID id_from_index(size_t index); @@ -100,11 +102,8 @@ namespace Kernel handle_smp_messages(); } - static uintptr_t current_stack_bottom() { return read_gs_sized(offsetof(Processor, m_stack)); } - static uintptr_t current_stack_top() { return current_stack_bottom() + s_stack_size; } - - uintptr_t stack_bottom() const { return reinterpret_cast(m_stack); } - uintptr_t stack_top() const { return stack_bottom() + s_stack_size; } + vaddr_t stack_top_vaddr() const { return m_stack_vaddr + s_stack_size; } + paddr_t stack_top_paddr() const { return m_stack_paddr + s_stack_size; } static void set_thread_syscall_stack(vaddr_t vaddr) { write_gs_sized(offsetof(Processor, m_thread_syscall_stack), vaddr); } @@ -215,8 +214,9 @@ namespace Kernel Thread* m_sse_thread { nullptr }; - static constexpr size_t s_stack_size { 4096 }; - void* m_stack { nullptr }; + static constexpr size_t s_stack_size { PAGE_SIZE }; + vaddr_t m_stack_vaddr { 0 }; + paddr_t m_stack_paddr { 0 }; GDT* m_gdt { nullptr }; IDT* m_idt { nullptr }; diff --git a/kernel/kernel/APIC.cpp b/kernel/kernel/APIC.cpp index be2b5c0d..ee89d9a5 100644 --- a/kernel/kernel/APIC.cpp +++ b/kernel/kernel/APIC.cpp @@ -293,9 +293,25 @@ namespace Kernel dprintln("Trying to enable processor (lapic id {})", processor.apic_id); auto& proc = Kernel::Processor::create(ProcessorID(processor.apic_id)); + proc.allocate_stack(); + + struct ap_init_info_t + { + uintptr_t stack_paddr; + uintptr_t stack_vaddr; + uintptr_t prepare_paging; + uintptr_t page_table; + uintptr_t ready; + }; + PageTable::with_fast_page(ap_init_paddr, [&] { - PageTable::fast_page_as_sized(2) = kmalloc_paddr_of(proc.stack_top()).value(); - PageTable::fast_page_as_sized(13) = 0; + PageTable::fast_page_as(8) = { + .stack_paddr = static_cast(proc.stack_top_paddr()), + .stack_vaddr = proc.stack_top_vaddr(), + .prepare_paging = reinterpret_cast(&PageTable::enable_cpu_features), + .page_table = static_cast(PageTable::kernel().paddr()), + .ready = 0, + }; }); write_to_local_apic(LAPIC_ERROR_REG, 0x00); @@ -334,12 +350,9 @@ namespace Kernel // give processor upto 100 * 100 us + 200 us to boot PageTable::with_fast_page(ap_init_paddr, [&] { - for (int i = 0; i < 100; i++) - { - if (__atomic_load_n(&PageTable::fast_page_as_sized(13), __ATOMIC_SEQ_CST)) + for (int i = 0; i < 100; i++, udelay(100)) + if (__atomic_load_n(&PageTable::fast_page_as(8).ready, __ATOMIC_SEQ_CST)) break; - udelay(100); - } }); initialized_aps++; diff --git a/kernel/kernel/Processor.cpp b/kernel/kernel/Processor.cpp index e110f224..17e34a3a 100644 --- a/kernel/kernel/Processor.cpp +++ b/kernel/kernel/Processor.cpp @@ -73,9 +73,6 @@ namespace Kernel ASSERT(processor.m_id == PROCESSOR_NONE); processor.m_id = id; - processor.m_stack = kmalloc(s_stack_size, 4096, true); - ASSERT(processor.m_stack); - processor.m_gdt = GDT::create(&processor); ASSERT(processor.m_gdt); @@ -157,6 +154,21 @@ namespace Kernel return processor; } + // NOTE: I don't like this being a separate function but we need heap and page tables for this :) + void Processor::allocate_stack() + { + ASSERT(m_stack_paddr == 0); + ASSERT(m_stack_vaddr == 0); + + m_stack_paddr = Heap::get().take_free_page(); + ASSERT(m_stack_paddr); + + m_stack_vaddr = PageTable::kernel().reserve_free_page(KERNEL_OFFSET); + ASSERT(m_stack_vaddr); + + PageTable::kernel().map_page_at(m_stack_paddr, m_stack_vaddr, PageTable::ReadWrite | PageTable::Present); + } + void Processor::initialize_smp() { const auto processor_id = current_id(); @@ -565,7 +577,7 @@ namespace Kernel if (!scheduler().is_idle()) Thread::current().set_cpu_time_stop(); - asm_yield_trampoline(Processor::current_stack_top()); + asm_yield_trampoline(processor_info.stack_top_vaddr()); processor_info.m_start_ns = SystemTimer::get().ns_since_boot(); diff --git a/kernel/kernel/kernel.cpp b/kernel/kernel/kernel.cpp index a6776ae2..03de6f3c 100644 --- a/kernel/kernel/kernel.cpp +++ b/kernel/kernel/kernel.cpp @@ -126,19 +126,20 @@ extern "C" void kernel_main(uint32_t boot_magic, uint32_t boot_info) parse_boot_info(boot_magic, boot_info); dprintln("boot info parsed"); - Processor::create(PROCESSOR_NONE); + auto& processor = Processor::create(PROCESSOR_NONE); Processor::initialize(); dprintln("BSP initialized"); - PageTable::initialize_pre_heap(); - PageTable::kernel().initial_load(); - dprintln("PageTable stage1 initialized"); + PageTable::initialize_fast_page(); + dprintln("fast page initialized"); Heap::initialize(); - dprintln("Heap initialzed"); + dprintln("Heap initialized"); - PageTable::initialize_post_heap(); - dprintln("PageTable stage2 initialized"); + PageTable::initialize_and_load(); + dprintln("PageTable initialized"); + + processor.allocate_stack(); parse_command_line(); dprintln("command line parsed, root='{}', console='{}'", cmdline.root, cmdline.console); @@ -270,7 +271,7 @@ extern "C" void ap_main() using namespace Kernel; Processor::initialize(); - PageTable::kernel().initial_load(); + InterruptController::get().enable(); Processor::wait_until_processors_ready();