Bananymous/banan-os
1
1
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Kernel: Rewrite paging and AP initialization

Browse Source 
Initial step of paging now just prepares fast page for heap, actual page
table initialization happens after heap is initialized which allows
x86_64 to never depend on kmalloc for pages.

Processor's stacks are now also spawned with PMM/VMM allocated stacks
instead of kmalloc identity mapped.
This commit is contained in:
Bananymous
2026-05-02 15:45:08 +03:00
parent 1602b195c5
commit d2b9b49cb0
11 changed files with 370 additions and 487 deletions
Download Patch File Download Diff File Expand all files Collapse all files

213
kernel/arch/i686/PageTable.cpp
View File

@@ -16,6 +16,8 @@ extern uint8_t g_kernel_writable_end[];
extern uint8_t g_userspace_start[];
extern uint8_t g_userspace_end[];
extern uint64_t g_boot_fast_page_pt[];
namespace Kernel
{
@@ -24,7 +26,7 @@ namespace Kernel
constexpr uint64_t s_page_flag_mask = 0x8000000000000FFF;
constexpr uint64_t s_page_addr_mask = ~s_page_flag_mask;
static bool s_is_post_heap_done = false;
static bool s_is_initialized = false;
static PageTable* s_kernel = nullptr;
static bool s_has_nxe = false;
@@ -33,6 +35,28 @@ namespace Kernel
static paddr_t s_global_pdpte = 0;
static uint64_t* s_fast_page_pt { nullptr };
static uint64_t* allocate_zeroed_page_aligned_page()
{
void* page = kmalloc(PAGE_SIZE, PAGE_SIZE, true);
ASSERT(page);
memset(page, 0, PAGE_SIZE);
return (uint64_t*)page;
}
template<typename T>
static paddr_t V2P(const T vaddr)
{
return (vaddr_t)vaddr - KERNEL_OFFSET + g_boot_info.kernel_paddr;
}
template<typename T>
static uint64_t* P2V(const T paddr)
{
return reinterpret_cast<uint64_t*>(reinterpret_cast<paddr_t>(paddr) - g_boot_info.kernel_paddr + KERNEL_OFFSET);
}
static inline PageTable::flags_t parse_flags(uint64_t entry)
{
using Flags = PageTable::Flags;
@@ -51,31 +75,22 @@ namespace Kernel
return result;
}
void PageTable::initialize_pre_heap()
void PageTable::initialize_fast_page()
{
s_fast_page_pt = g_boot_fast_page_pt;
}
static void detect_cpu_features()
{
if (CPUID::has_nxe())
s_has_nxe = true;
if (CPUID::has_pge())
s_has_pge = true;
if (CPUID::has_pat())
s_has_pat = true;
ASSERT(s_kernel == nullptr);
s_kernel = new PageTable();
ASSERT(s_kernel);
s_kernel->initialize_kernel();
s_kernel->initial_load();
}
void PageTable::initialize_post_heap()
{
s_is_post_heap_done = true;
}
void PageTable::initial_load()
void PageTable::enable_cpu_features()
{
if (s_has_nxe)
{
@@ -116,8 +131,56 @@ namespace Kernel
"movl %%eax, %%cr0;"
::: "rax"
);
}
load();
void PageTable::initialize_and_load()
{
detect_cpu_features();
enable_cpu_features();
ASSERT(s_kernel == nullptr);
s_kernel = new PageTable();
ASSERT(s_kernel);
auto* pdpt = allocate_zeroed_page_aligned_page();
ASSERT(pdpt);
s_kernel->m_highest_paging_struct = V2P(pdpt);
s_kernel->map_kernel_memory();
PageTable::with_fast_page(s_kernel->m_highest_paging_struct, [] {
s_global_pdpte = PageTable::fast_page_as_sized<paddr_t>(3);
});
// update fast page pt
{
constexpr vaddr_t vaddr = fast_page();
constexpr uint16_t pdpte = (vaddr >> 30) & 0x1FF;
constexpr uint16_t pde = (vaddr >> 21) & 0x1FF;
const auto get_or_allocate_entry =
[](paddr_t table_paddr, uint16_t entry, uint64_t flags)
{
uint64_t* table = P2V(table_paddr);
if (!(table[entry] & Flags::Present))
{
auto* vaddr = allocate_zeroed_page_aligned_page();
ASSERT(vaddr);
table[entry] = V2P(vaddr);
}
table[entry] |= flags;
return table[entry] & s_page_addr_mask;
};
const paddr_t pdpt = s_kernel->m_highest_paging_struct;
const paddr_t pd = get_or_allocate_entry(pdpt, pdpte, Flags::Present);
s_fast_page_pt = P2V(get_or_allocate_entry(pd, pde, Flags::ReadWrite | Flags::Present));
}
s_kernel->load();
}
PageTable& PageTable::kernel()
@@ -131,40 +194,12 @@ namespace Kernel
return true;
}
static uint64_t* allocate_zeroed_page_aligned_page()
void PageTable::map_kernel_memory()
{
void* page = kmalloc(PAGE_SIZE, PAGE_SIZE, true);
ASSERT(page);
memset(page, 0, PAGE_SIZE);
return (uint64_t*)page;
}
template<typename T>
static paddr_t V2P(const T vaddr)
{
return (vaddr_t)vaddr - KERNEL_OFFSET + g_boot_info.kernel_paddr;
}
template<typename T>
static uint64_t* P2V(const T paddr)
{
return reinterpret_cast<uint64_t*>(reinterpret_cast<paddr_t>(paddr) - g_boot_info.kernel_paddr + KERNEL_OFFSET);
}
void PageTable::initialize_kernel()
{
ASSERT(s_global_pdpte == 0);
s_global_pdpte = V2P(allocate_zeroed_page_aligned_page());
map_kernel_memory();
prepare_fast_page();
// Map (phys_kernel_start -> phys_kernel_end) to (virt_kernel_start -> virt_kernel_end)
ASSERT((vaddr_t)g_kernel_start % PAGE_SIZE == 0);
map_range_at(
V2P(g_kernel_start),
(vaddr_t)g_kernel_start,
reinterpret_cast<vaddr_t>(g_kernel_start),
g_kernel_end - g_kernel_start,
Flags::Present
);
@@ -172,7 +207,7 @@ namespace Kernel
// Map executable kernel memory as executable
map_range_at(
V2P(g_kernel_execute_start),
(vaddr_t)g_kernel_execute_start,
reinterpret_cast<vaddr_t>(g_kernel_execute_start),
g_kernel_execute_end - g_kernel_execute_start,
Flags::Execute | Flags::Present
);
@@ -180,7 +215,7 @@ namespace Kernel
// Map writable kernel memory as writable
map_range_at(
V2P(g_kernel_writable_start),
(vaddr_t)g_kernel_writable_start,
reinterpret_cast<vaddr_t>(g_kernel_writable_start),
g_kernel_writable_end - g_kernel_writable_start,
Flags::ReadWrite | Flags::Present
);
@@ -188,70 +223,34 @@ namespace Kernel
// Map userspace memory
map_range_at(
V2P(g_userspace_start),
(vaddr_t)g_userspace_start,
reinterpret_cast<vaddr_t>(g_userspace_start),
g_userspace_end - g_userspace_start,
Flags::Execute | Flags::UserSupervisor | Flags::Present
);
}
void PageTable::prepare_fast_page()
{
constexpr uint64_t pdpte = (fast_page() >> 30) & 0x1FF;
constexpr uint64_t pde = (fast_page() >> 21) & 0x1FF;
constexpr uint64_t pte = (fast_page() >> 12) & 0x1FF;
const uint64_t* pdpt = P2V(m_highest_paging_struct);
ASSERT(pdpt[pdpte] & Flags::Present);
uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
ASSERT(!(pd[pde] & Flags::Present));
pd[pde] = V2P(allocate_zeroed_page_aligned_page()) | Flags::ReadWrite | Flags::Present;
uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
ASSERT(pt[pte] == 0);
pt[pte] = Flags::Reserved;
}
void PageTable::map_fast_page(paddr_t paddr)
{
ASSERT(s_kernel);
ASSERT(paddr);
ASSERT(paddr % PAGE_SIZE == 0);
ASSERT(paddr && paddr % PAGE_SIZE == 0);
ASSERT(s_fast_page_pt);
ASSERT(s_fast_page_lock.current_processor_has_lock());
constexpr uint64_t pdpte = (fast_page() >> 30) & 0x1FF;
constexpr uint64_t pde = (fast_page() >> 21) & 0x1FF;
constexpr uint64_t pte = (fast_page() >> 12) & 0x1FF;
ASSERT(!(*s_fast_page_pt & Flags::Present));
s_fast_page_pt[0] = paddr | Flags::ReadWrite | Flags::Present;
uint64_t* pdpt = P2V(s_kernel->m_highest_paging_struct);
uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
ASSERT(!(pt[pte] & Flags::Present));
pt[pte] = paddr | Flags::ReadWrite | Flags::Present;
asm volatile("invlpg (%0)" :: "r"(fast_page()) : "memory");
asm volatile("invlpg (%0)" :: "r"(fast_page()));
}
void PageTable::unmap_fast_page()
{
ASSERT(s_kernel);
ASSERT(s_fast_page_pt);
ASSERT(s_fast_page_lock.current_processor_has_lock());
constexpr uint64_t pdpte = (fast_page() >> 30) & 0x1FF;
constexpr uint64_t pde = (fast_page() >> 21) & 0x1FF;
constexpr uint64_t pte = (fast_page() >> 12) & 0x1FF;
ASSERT((*s_fast_page_pt & Flags::Present));
s_fast_page_pt[0] = 0;
uint64_t* pdpt = P2V(s_kernel->m_highest_paging_struct);
uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
ASSERT(pt[pte] & Flags::Present);
pt[pte] = Flags::Reserved;
asm volatile("invlpg (%0)" :: "r"(fast_page()) : "memory");
asm volatile("invlpg (%0)" :: "r"(fast_page()));
}
BAN::ErrorOr<PageTable*> PageTable::create_userspace()
@@ -260,25 +259,23 @@ namespace Kernel
PageTable* page_table = new PageTable;
if (page_table == nullptr)
return BAN::Error::from_errno(ENOMEM);
page_table->map_kernel_memory();
return page_table;
}
void PageTable::map_kernel_memory()
{
ASSERT(s_kernel);
ASSERT(s_global_pdpte);
uint64_t* pdpt = allocate_zeroed_page_aligned_page();
if (pdpt == nullptr)
{
delete page_table;
return BAN::Error::from_errno(ENOMEM);
}
ASSERT(m_highest_paging_struct == 0);
m_highest_paging_struct = V2P(kmalloc(32, 32, true));
ASSERT(m_highest_paging_struct);
page_table->m_highest_paging_struct = V2P(pdpt);
uint64_t* pdpt = P2V(m_highest_paging_struct);
pdpt[0] = 0;
pdpt[1] = 0;
pdpt[2] = 0;
pdpt[3] = s_global_pdpte | Flags::Present;
static_assert(KERNEL_OFFSET == 0xC0000000);
return page_table;
}
PageTable::~PageTable()
@@ -318,7 +315,7 @@ namespace Kernel
const bool is_userspace = (vaddr < KERNEL_OFFSET);
if (is_userspace && this != &PageTable::current())
;
else if (pages <= 32 || !s_is_post_heap_done)
else if (pages <= 32 || !s_is_initialized)
{
for (size_t i = 0; i < pages; i++, vaddr += PAGE_SIZE)
asm volatile("invlpg (%0)" :: "r"(vaddr));

51
kernel/arch/i686/boot.S
View File

@@ -98,8 +98,7 @@ bananboot_end:
boot_pdpt:
.long V2P(boot_pd) + (PG_PRESENT)
.long 0
.quad 0
.quad 0
.skip 2 * 8
.long V2P(boot_pd) + (PG_PRESENT)
.long 0
.align 4096
@@ -112,13 +111,16 @@ boot_pd:
.endr
boot_pts:
.set i, 0
.rept 512
.rept 511
.rept 512
.long i + (PG_READ_WRITE | PG_PRESENT)
.long 0
.set i, i + 0x1000
.endr
.endr
.global g_boot_fast_page_pt
g_boot_fast_page_pt:
.skip 512 * 8
boot_gdt:
.quad 0x0000000000000000 # null descriptor
@@ -274,7 +276,7 @@ system_halt:
jmp 1b
#define AP_V2P(vaddr) ((vaddr) - ap_trampoline + 0xF000)
#define AP_REL(vaddr) ((vaddr) - ap_trampoline + 0xF000)
.section .ap_init, "ax"
@@ -284,21 +286,27 @@ ap_trampoline:
jmp 1f
.align 8
ap_stack_ptr:
ap_stack_paddr:
.skip 4
ap_stack_vaddr:
.skip 4
ap_prepare_paging:
.skip 4
ap_page_table:
.skip 4
ap_ready:
.skip 4
ap_stack_loaded:
.skip 1
1: cli; cld
ljmpl $0x00, $AP_V2P(ap_cs_clear)
ljmpl $0x00, $AP_REL(ap_cs_clear)
ap_cs_clear:
# load ap gdt and enter protected mode
lgdt AP_V2P(ap_gdtr)
lgdt AP_REL(ap_gdtr)
movl %cr0, %eax
orb $1, %al
movl %eax, %cr0
ljmpl $0x08, $AP_V2P(ap_protected_mode)
ljmpl $0x08, $AP_REL(ap_protected_mode)
.code32
ap_protected_mode:
@@ -307,8 +315,7 @@ ap_protected_mode:
movw %ax, %ss
movw %ax, %es
movl AP_V2P(ap_stack_ptr), %esp
movb $1, AP_V2P(ap_stack_loaded)
movl AP_REL(ap_stack_paddr), %esp
leal V2P(enable_sse), %ecx; call *%ecx
leal V2P(enable_tsc), %ecx; call *%ecx
@@ -316,24 +323,28 @@ ap_protected_mode:
# load boot gdt and enter long mode
lgdt V2P(boot_gdtr)
ljmpl $0x08, $AP_V2P(ap_flush_gdt)
ljmpl $0x08, $AP_REL(ap_flush_gdt)
ap_flush_gdt:
# move stack pointer to higher half
movl %esp, %esp
addl $KERNEL_OFFSET, %esp
# jump to higher half
leal ap_higher_half, %ecx
movl $ap_higher_half, %ecx
jmp *%ecx
ap_higher_half:
movl AP_REL(ap_prepare_paging), %eax
call *%eax
# load AP's initial values
movl AP_REL(ap_stack_vaddr), %esp
movl AP_REL(ap_page_table), %eax
movl $1, AP_REL(ap_ready)
movl %eax, %cr3
# clear rbp for stacktrace
xorl %ebp, %ebp
1: pause
cmpb $0, g_ap_startup_done
jz 1b
je 1b
lock incb g_ap_running_count

12
kernel/arch/i686/linker.ld
View File

@@ -15,17 +15,17 @@ SECTIONS
*(.bananboot)
*(.text.*)
}
.ap_init ALIGN(4K) : AT(ADDR(.ap_init) - KERNEL_OFFSET)
{
g_ap_init_addr = .;
*(.ap_init)
g_kernel_execute_end = .;
}
.userspace ALIGN(4K) : AT(ADDR(.userspace) - KERNEL_OFFSET)
{
g_userspace_start = .;
*(.userspace)
g_userspace_end = .;
g_kernel_execute_end = .;
}
.ap_init ALIGN(4K) : AT(ADDR(.ap_init) - KERNEL_OFFSET)
{
g_ap_init_addr = .;
*(.ap_init)
}
.rodata ALIGN(4K) : AT(ADDR(.rodata) - KERNEL_OFFSET)
{