Bananymous/banan-os
1
1
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Kernel: Cleanup and fix page tables and better TLB shootdown

Browse Source
This commit is contained in:
Bananymous
2026-04-03 01:53:30 +03:00
parent f77aa65dc5
commit 7d8f7753d5
4 changed files with 170 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files

182
kernel/arch/i686/PageTable.cpp
View File

@@ -21,6 +21,11 @@ namespace Kernel
SpinLock PageTable::s_fast_page_lock; SpinLock PageTable::s_fast_page_lock;
constexpr uint64_t s_page_flag_mask = 0x8000000000000FFF;
constexpr uint64_t s_page_addr_mask = ~s_page_flag_mask;
static bool s_is_post_heap_done = false;
static PageTable* s_kernel = nullptr; static PageTable* s_kernel = nullptr;
static bool s_has_nxe = false; static bool s_has_nxe = false;
static bool s_has_pge = false; static bool s_has_pge = false;
@@ -67,7 +72,7 @@ namespace Kernel
void PageTable::initialize_post_heap() void PageTable::initialize_post_heap()
{ {
// NOTE: this is no-op as our 32 bit target does not use hhdm s_is_post_heap_done = true;
} }
void PageTable::initial_load() void PageTable::initial_load()
@@ -141,9 +146,9 @@ namespace Kernel
} }
template<typename T> template<typename T>
static vaddr_t P2V(const T paddr) static uint64_t* P2V(const T paddr)
{ {
return (paddr_t)paddr - g_boot_info.kernel_paddr + KERNEL_OFFSET; return reinterpret_cast<uint64_t*>(reinterpret_cast<paddr_t>(paddr) - g_boot_info.kernel_paddr + KERNEL_OFFSET);
} }
void PageTable::initialize_kernel() void PageTable::initialize_kernel()
@@ -194,10 +199,10 @@ namespace Kernel
constexpr uint64_t pdpte = (fast_page() >> 30) & 0x1FF; constexpr uint64_t pdpte = (fast_page() >> 30) & 0x1FF;
constexpr uint64_t pde = (fast_page() >> 21) & 0x1FF; constexpr uint64_t pde = (fast_page() >> 21) & 0x1FF;
uint64_t* pdpt = reinterpret_cast<uint64_t*>(P2V(m_highest_paging_struct)); const uint64_t* pdpt = P2V(m_highest_paging_struct);
ASSERT(pdpt[pdpte] & Flags::Present); ASSERT(pdpt[pdpte] & Flags::Present);
uint64_t* pd = reinterpret_cast<uint64_t*>(P2V(pdpt[pdpte]) & PAGE_ADDR_MASK); uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
ASSERT(!(pd[pde] & Flags::Present)); ASSERT(!(pd[pde] & Flags::Present));
pd[pde] = V2P(allocate_zeroed_page_aligned_page()) | Flags::ReadWrite | Flags::Present; pd[pde] = V2P(allocate_zeroed_page_aligned_page()) | Flags::ReadWrite | Flags::Present;
} }
@@ -214,9 +219,9 @@ namespace Kernel
constexpr uint64_t pde = (fast_page() >> 21) & 0x1FF; constexpr uint64_t pde = (fast_page() >> 21) & 0x1FF;
constexpr uint64_t pte = (fast_page() >> 12) & 0x1FF; constexpr uint64_t pte = (fast_page() >> 12) & 0x1FF;
uint64_t* pdpt = reinterpret_cast<uint64_t*>(P2V(s_kernel->m_highest_paging_struct)); uint64_t* pdpt = P2V(s_kernel->m_highest_paging_struct);
uint64_t* pd = reinterpret_cast<uint64_t*>(P2V(pdpt[pdpte] & PAGE_ADDR_MASK)); uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
uint64_t* pt = reinterpret_cast<uint64_t*>(P2V(pd[pde] & PAGE_ADDR_MASK)); uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
ASSERT(!(pt[pte] & Flags::Present)); ASSERT(!(pt[pte] & Flags::Present));
pt[pte] = paddr | Flags::ReadWrite | Flags::Present; pt[pte] = paddr | Flags::ReadWrite | Flags::Present;
@@ -234,9 +239,9 @@ namespace Kernel
constexpr uint64_t pde = (fast_page() >> 21) & 0x1FF; constexpr uint64_t pde = (fast_page() >> 21) & 0x1FF;
constexpr uint64_t pte = (fast_page() >> 12) & 0x1FF; constexpr uint64_t pte = (fast_page() >> 12) & 0x1FF;
uint64_t* pdpt = reinterpret_cast<uint64_t*>(P2V(s_kernel->m_highest_paging_struct)); uint64_t* pdpt = P2V(s_kernel->m_highest_paging_struct);
uint64_t* pd = reinterpret_cast<uint64_t*>(P2V(pdpt[pdpte] & PAGE_ADDR_MASK)); uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
uint64_t* pt = reinterpret_cast<uint64_t*>(P2V(pd[pde] & PAGE_ADDR_MASK)); uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
ASSERT(pt[pte] & Flags::Present); ASSERT(pt[pte] & Flags::Present);
pt[pte] = 0; pt[pte] = 0;
@@ -263,7 +268,7 @@ namespace Kernel
m_highest_paging_struct = V2P(kmalloc(32, 32, true)); m_highest_paging_struct = V2P(kmalloc(32, 32, true));
ASSERT(m_highest_paging_struct); ASSERT(m_highest_paging_struct);
uint64_t* pdpt = reinterpret_cast<uint64_t*>(P2V(m_highest_paging_struct)); uint64_t* pdpt = P2V(m_highest_paging_struct);
pdpt[0] = 0; pdpt[0] = 0;
pdpt[1] = 0; pdpt[1] = 0;
pdpt[2] = 0; pdpt[2] = 0;
@@ -276,18 +281,17 @@ namespace Kernel
if (m_highest_paging_struct == 0) if (m_highest_paging_struct == 0)
return; return;
uint64_t* pdpt = reinterpret_cast<uint64_t*>(P2V(m_highest_paging_struct)); uint64_t* pdpt = P2V(m_highest_paging_struct);
for (uint32_t pdpte = 0; pdpte < 3; pdpte++) for (uint32_t pdpte = 0; pdpte < 3; pdpte++)
{ {
if (!(pdpt[pdpte] & Flags::Present)) if (!(pdpt[pdpte] & Flags::Present))
continue; continue;
uint64_t* pd = reinterpret_cast<uint64_t*>(P2V(pdpt[pdpte] & PAGE_ADDR_MASK)); uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
for (uint32_t pde = 0; pde < 512; pde++) for (uint32_t pde = 0; pde < 512; pde++)
{ {
if (!(pd[pde] & Flags::Present)) if (!(pd[pde] & Flags::Present))
continue; continue;
kfree(reinterpret_cast<uint64_t*>(P2V(pd[pde] & PAGE_ADDR_MASK))); kfree(P2V(pd[pde] & s_page_addr_mask));
} }
kfree(pd); kfree(pd);
} }
@@ -298,15 +302,43 @@ namespace Kernel
{ {
SpinLockGuard _(m_lock); SpinLockGuard _(m_lock);
ASSERT(m_highest_paging_struct < 0x100000000); ASSERT(m_highest_paging_struct < 0x100000000);
const uint32_t pdpt_lo = m_highest_paging_struct; asm volatile("movl %0, %%cr3" :: "r"(static_cast<uint32_t>(m_highest_paging_struct)));
asm volatile("movl %0, %%cr3" :: "r"(pdpt_lo));
Processor::set_current_page_table(this); Processor::set_current_page_table(this);
} }
void PageTable::invalidate(vaddr_t vaddr, bool send_smp_message) void PageTable::invalidate_range(vaddr_t vaddr, size_t pages, bool send_smp_message)
{ {
ASSERT(vaddr % PAGE_SIZE == 0); ASSERT(vaddr % PAGE_SIZE == 0);
asm volatile("invlpg (%0)" :: "r"(vaddr) : "memory");
const bool is_userspace = (vaddr < KERNEL_OFFSET);
if (is_userspace && this != &PageTable::current())
;
else if (pages <= 32 || !s_is_post_heap_done)
{
for (size_t i = 0; i < pages; i++, vaddr += PAGE_SIZE)
asm volatile("invlpg (%0)" :: "r"(vaddr));
}
else if (is_userspace || !s_has_pge)
{
asm volatile("movl %0, %%cr3" :: "r"(static_cast<uint32_t>(m_highest_paging_struct)));
}
else
{
asm volatile(
"movl %%cr4, %%eax;"
"andl $~0x80, %%eax;"
"movl %%eax, %%cr4;"
"movl %0, %%cr3;"
"orl $0x80, %%eax;"
"movl %%eax, %%cr4;"
:
: "r"(static_cast<uint32_t>(m_highest_paging_struct))
: "eax"
);
}
if (send_smp_message) if (send_smp_message)
{ {
@@ -314,14 +346,14 @@ namespace Kernel
.type = Processor::SMPMessage::Type::FlushTLB, .type = Processor::SMPMessage::Type::FlushTLB,
.flush_tlb = { .flush_tlb = {
.vaddr = vaddr, .vaddr = vaddr,
.page_count = 1, .page_count = pages,
.page_table = vaddr < KERNEL_OFFSET ? this : nullptr, .page_table = vaddr < KERNEL_OFFSET ? this : nullptr,
} }
}); });
} }
} }
void PageTable::unmap_page(vaddr_t vaddr, bool send_smp_message) void PageTable::unmap_page(vaddr_t vaddr, bool invalidate)
{ {
ASSERT(vaddr); ASSERT(vaddr);
ASSERT(vaddr % PAGE_SIZE == 0); ASSERT(vaddr % PAGE_SIZE == 0);
@@ -340,16 +372,16 @@ namespace Kernel
if (is_page_free(vaddr)) if (is_page_free(vaddr))
Kernel::panic("trying to unmap unmapped page 0x{H}", vaddr); Kernel::panic("trying to unmap unmapped page 0x{H}", vaddr);
uint64_t* pdpt = reinterpret_cast<uint64_t*>(P2V(m_highest_paging_struct)); uint64_t* pdpt = P2V(m_highest_paging_struct);
uint64_t* pd = reinterpret_cast<uint64_t*>(P2V(pdpt[pdpte] & PAGE_ADDR_MASK)); uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
uint64_t* pt = reinterpret_cast<uint64_t*>(P2V(pd[pde] & PAGE_ADDR_MASK)); uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
const paddr_t old_paddr = pt[pte] & PAGE_ADDR_MASK; const paddr_t old_paddr = pt[pte] & s_page_addr_mask;
pt[pte] = 0; pt[pte] = 0;
if (old_paddr != 0) if (invalidate && old_paddr != 0)
invalidate(vaddr, send_smp_message); invalidate_page(vaddr, true);
} }
void PageTable::unmap_range(vaddr_t vaddr, size_t size) void PageTable::unmap_range(vaddr_t vaddr, size_t size)
@@ -361,18 +393,10 @@ namespace Kernel
SpinLockGuard _(m_lock); SpinLockGuard _(m_lock);
for (vaddr_t page = 0; page < page_count; page++) for (vaddr_t page = 0; page < page_count; page++)
unmap_page(vaddr + page * PAGE_SIZE, false); unmap_page(vaddr + page * PAGE_SIZE, false);
invalidate_range(vaddr, page_count, true);
Processor::broadcast_smp_message({
.type = Processor::SMPMessage::Type::FlushTLB,
.flush_tlb = {
.vaddr = vaddr,
.page_count = page_count,
.page_table = vaddr < KERNEL_OFFSET ? this : nullptr,
}
});
} }
void PageTable::map_page_at(paddr_t paddr, vaddr_t vaddr, flags_t flags, MemoryType memory_type, bool send_smp_message) void PageTable::map_page_at(paddr_t paddr, vaddr_t vaddr, flags_t flags, MemoryType memory_type, bool invalidate)
{ {
ASSERT(vaddr); ASSERT(vaddr);
ASSERT(vaddr != fast_page()); ASSERT(vaddr != fast_page());
@@ -407,11 +431,11 @@ namespace Kernel
SpinLockGuard _(m_lock); SpinLockGuard _(m_lock);
uint64_t* pdpt = reinterpret_cast<uint64_t*>(P2V(m_highest_paging_struct)); uint64_t* pdpt = P2V(m_highest_paging_struct);
if (!(pdpt[pdpte] & Flags::Present)) if (!(pdpt[pdpte] & Flags::Present))
pdpt[pdpte] = V2P(allocate_zeroed_page_aligned_page()) | Flags::Present; pdpt[pdpte] = V2P(allocate_zeroed_page_aligned_page()) | Flags::Present;
uint64_t* pd = reinterpret_cast<uint64_t*>(P2V(pdpt[pdpte] & PAGE_ADDR_MASK)); uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
if ((pd[pde] & uwr_flags) != uwr_flags) if ((pd[pde] & uwr_flags) != uwr_flags)
{ {
if (!(pd[pde] & Flags::Present)) if (!(pd[pde] & Flags::Present))
@@ -422,14 +446,14 @@ namespace Kernel
if (!(flags & Flags::Present)) if (!(flags & Flags::Present))
uwr_flags &= ~Flags::Present; uwr_flags &= ~Flags::Present;
uint64_t* pt = reinterpret_cast<uint64_t*>(P2V(pd[pde] & PAGE_ADDR_MASK)); uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
const paddr_t old_paddr = pt[pte] & PAGE_ADDR_MASK; const paddr_t old_paddr = pt[pte] & s_page_addr_mask;
pt[pte] = paddr | uwr_flags | extra_flags; pt[pte] = paddr | uwr_flags | extra_flags;
if (old_paddr != 0) if (invalidate && old_paddr != 0)
invalidate(vaddr, send_smp_message); invalidate_page(vaddr, true);
} }
void PageTable::map_range_at(paddr_t paddr, vaddr_t vaddr, size_t size, flags_t flags, MemoryType memory_type) void PageTable::map_range_at(paddr_t paddr, vaddr_t vaddr, size_t size, flags_t flags, MemoryType memory_type)
@@ -443,15 +467,7 @@ namespace Kernel
SpinLockGuard _(m_lock); SpinLockGuard _(m_lock);
for (size_t page = 0; page < page_count; page++) for (size_t page = 0; page < page_count; page++)
map_page_at(paddr + page * PAGE_SIZE, vaddr + page * PAGE_SIZE, flags, memory_type, false); map_page_at(paddr + page * PAGE_SIZE, vaddr + page * PAGE_SIZE, flags, memory_type, false);
invalidate_range(vaddr, page_count, true);
Processor::broadcast_smp_message({
.type = Processor::SMPMessage::Type::FlushTLB,
.flush_tlb = {
.vaddr = vaddr,
.page_count = page_count,
.page_table = vaddr < KERNEL_OFFSET ? this : nullptr,
}
});
} }
uint64_t PageTable::get_page_data(vaddr_t vaddr) const uint64_t PageTable::get_page_data(vaddr_t vaddr) const
@@ -464,15 +480,15 @@ namespace Kernel
SpinLockGuard _(m_lock); SpinLockGuard _(m_lock);
uint64_t* pdpt = (uint64_t*)P2V(m_highest_paging_struct); const uint64_t* pdpt = P2V(m_highest_paging_struct);
if (!(pdpt[pdpte] & Flags::Present)) if (!(pdpt[pdpte] & Flags::Present))
return 0; return 0;
uint64_t* pd = (uint64_t*)P2V(pdpt[pdpte] & PAGE_ADDR_MASK); const uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
if (!(pd[pde] & Flags::Present)) if (!(pd[pde] & Flags::Present))
return 0; return 0;
uint64_t* pt = (uint64_t*)P2V(pd[pde] & PAGE_ADDR_MASK); const uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
if (!(pt[pte] & Flags::Used)) if (!(pt[pte] & Flags::Used))
return 0; return 0;
@@ -486,8 +502,7 @@ namespace Kernel
paddr_t PageTable::physical_address_of(vaddr_t vaddr) const paddr_t PageTable::physical_address_of(vaddr_t vaddr) const
{ {
uint64_t page_data = get_page_data(vaddr); return get_page_data(vaddr) & s_page_addr_mask;
return (page_data & PAGE_ADDR_MASK) & ~(1ull << 63);
} }
bool PageTable::is_page_free(vaddr_t vaddr) const bool PageTable::is_page_free(vaddr_t vaddr) const
@@ -529,14 +544,8 @@ namespace Kernel
return false; return false;
for (size_t offset = 0; offset < bytes; offset += PAGE_SIZE) for (size_t offset = 0; offset < bytes; offset += PAGE_SIZE)
reserve_page(vaddr + offset, true, false); reserve_page(vaddr + offset, true, false);
Processor::broadcast_smp_message({ invalidate_range(vaddr, bytes / PAGE_SIZE, true);
.type = Processor::SMPMessage::Type::FlushTLB,
.flush_tlb = {
.vaddr = vaddr,
.page_count = bytes / PAGE_SIZE,
.page_table = vaddr < KERNEL_OFFSET ? this : nullptr,
}
});
return true; return true;
} }
@@ -549,39 +558,37 @@ namespace Kernel
if (size_t rem = last_address % PAGE_SIZE) if (size_t rem = last_address % PAGE_SIZE)
last_address -= rem; last_address -= rem;
const uint32_t s_pdpte = (first_address >> 30) & 0x1FF; uint32_t pdpte = (first_address >> 30) & 0x1FF;
const uint32_t s_pde = (first_address >> 21) & 0x1FF; uint32_t pde = (first_address >> 21) & 0x1FF;
const uint32_t s_pte = (first_address >> 12) & 0x1FF; uint32_t pte = (first_address >> 12) & 0x1FF;
const uint32_t e_pdpte = (last_address >> 30) & 0x1FF; const uint32_t e_pdpte = ((last_address - 1) >> 30) & 0x1FF;
const uint32_t e_pde = (last_address >> 21) & 0x1FF; const uint32_t e_pde = ((last_address - 1) >> 21) & 0x1FF;
const uint32_t e_pte = (last_address >> 12) & 0x1FF; const uint32_t e_pte = ((last_address - 1) >> 12) & 0x1FF;
SpinLockGuard _(m_lock); SpinLockGuard _(m_lock);
// Try to find free page that can be mapped without // Try to find free page that can be mapped without
// allocations (page table with unused entries) // allocations (page table with unused entries)
uint64_t* pdpt = reinterpret_cast<uint64_t*>(P2V(m_highest_paging_struct)); const uint64_t* pdpt = P2V(m_highest_paging_struct);
for (uint32_t pdpte = s_pdpte; pdpte < 4; pdpte++) for (; pdpte <= e_pdpte; pdpte++)
{ {
if (pdpte > e_pdpte)
break;
if (!(pdpt[pdpte] & Flags::Present)) if (!(pdpt[pdpte] & Flags::Present))
continue; continue;
uint64_t* pd = reinterpret_cast<uint64_t*>(P2V(pdpt[pdpte] & PAGE_ADDR_MASK)); const uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
for (uint32_t pde = s_pde; pde < 512; pde++) for (; pde < 512; pde++)
{ {
if (pdpte == e_pdpte && pde > e_pde) if (pdpte == e_pdpte && pde > e_pde)
break; break;
if (!(pd[pde] & Flags::Present)) if (!(pd[pde] & Flags::Present))
continue; continue;
uint64_t* pt = (uint64_t*)P2V(pd[pde] & PAGE_ADDR_MASK); const uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
for (uint32_t pte = s_pte; pte < 512; pte++) for (; pte < 512; pte++)
{ {
if (pdpte == e_pdpte && pde == e_pde && pte >= e_pte) if (pdpte == e_pdpte && pde == e_pde && pte > e_pte)
break; break;
if (!(pt[pte] & Flags::Used)) if (pt[pte] & Flags::Used)
{ continue;
vaddr_t vaddr = 0; vaddr_t vaddr = 0;
vaddr |= (vaddr_t)pdpte << 30; vaddr |= (vaddr_t)pdpte << 30;
vaddr |= (vaddr_t)pde << 21; vaddr |= (vaddr_t)pde << 21;
@@ -589,8 +596,9 @@ namespace Kernel
ASSERT(reserve_page(vaddr)); ASSERT(reserve_page(vaddr));
return vaddr; return vaddr;
} }
pte = 0;
} }
} pde = 0;
} }
// Find any free page // Find any free page
@@ -659,7 +667,7 @@ namespace Kernel
flags_t flags = 0; flags_t flags = 0;
vaddr_t start = 0; vaddr_t start = 0;
uint64_t* pdpt = reinterpret_cast<uint64_t*>(P2V(m_highest_paging_struct)); const uint64_t* pdpt = P2V(m_highest_paging_struct);
for (uint32_t pdpte = 0; pdpte < 4; pdpte++) for (uint32_t pdpte = 0; pdpte < 4; pdpte++)
{ {
if (!(pdpt[pdpte] & Flags::Present)) if (!(pdpt[pdpte] & Flags::Present))
@@ -668,7 +676,7 @@ namespace Kernel
start = 0; start = 0;
continue; continue;
} }
uint64_t* pd = (uint64_t*)P2V(pdpt[pdpte] & PAGE_ADDR_MASK); const uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
for (uint64_t pde = 0; pde < 512; pde++) for (uint64_t pde = 0; pde < 512; pde++)
{ {
if (!(pd[pde] & Flags::Present)) if (!(pd[pde] & Flags::Present))
@@ -677,7 +685,7 @@ namespace Kernel
start = 0; start = 0;
continue; continue;
} }
uint64_t* pt = (uint64_t*)P2V(pd[pde] & PAGE_ADDR_MASK); const uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
for (uint64_t pte = 0; pte < 512; pte++) for (uint64_t pte = 0; pte < 512; pte++)
{ {
if (parse_flags(pt[pte]) != flags) if (parse_flags(pt[pte]) != flags)

102
kernel/arch/x86_64/PageTable.cpp
View File

@@ -23,7 +23,7 @@ namespace Kernel
SpinLock PageTable::s_fast_page_lock; SpinLock PageTable::s_fast_page_lock;
static constexpr vaddr_t s_hhdm_offset = 0xFFFF800000000000; static constexpr vaddr_t s_hhdm_offset = 0xFFFF800000000000;
static bool s_is_hddm_initialized = false; static bool s_is_post_heap_done = false;
constexpr uint64_t s_page_flag_mask = 0x8000000000000FFF; constexpr uint64_t s_page_flag_mask = 0x8000000000000FFF;
constexpr uint64_t s_page_addr_mask = ~s_page_flag_mask; constexpr uint64_t s_page_addr_mask = ~s_page_flag_mask;
@@ -376,7 +376,7 @@ namespace Kernel
V2P = &FuncsHHDM::V2P; V2P = &FuncsHHDM::V2P;
P2V = &FuncsHHDM::P2V; P2V = &FuncsHHDM::P2V;
s_is_hddm_initialized = true; s_is_post_heap_done = true;
// This is a hack to unmap fast page. fast page pt is copied // This is a hack to unmap fast page. fast page pt is copied
// while it is mapped, so we need to manually unmap it // while it is mapped, so we need to manually unmap it
@@ -612,10 +612,39 @@ namespace Kernel
Processor::set_current_page_table(this); Processor::set_current_page_table(this);
} }
void PageTable::invalidate(vaddr_t vaddr, bool send_smp_message) void PageTable::invalidate_range(vaddr_t vaddr, size_t pages, bool send_smp_message)
{ {
ASSERT(vaddr % PAGE_SIZE == 0); ASSERT(vaddr % PAGE_SIZE == 0);
asm volatile("invlpg (%0)" :: "r"(vaddr) : "memory");
const bool is_userspace = (vaddr < KERNEL_OFFSET);
if (is_userspace && this != &PageTable::current())
;
else if (pages <= 32 || !s_is_post_heap_done)
{
for (size_t i = 0; i < pages; i++, vaddr += PAGE_SIZE)
asm volatile("invlpg (%0)" :: "r"(vaddr));
}
else if (is_userspace || !s_has_pge)
{
asm volatile("movq %0, %%cr3" :: "r"(m_highest_paging_struct));
}
else
{
asm volatile(
"movq %%cr4, %%rax;"
"andq $~0x80, %%rax;"
"movq %%rax, %%cr4;"
"movq %0, %%cr3;"
"orq $0x80, %%rax;"
"movq %%rax, %%cr4;"
:
: "r"(m_highest_paging_struct)
: "rax"
);
}
if (send_smp_message) if (send_smp_message)
{ {
@@ -623,14 +652,14 @@ namespace Kernel
.type = Processor::SMPMessage::Type::FlushTLB, .type = Processor::SMPMessage::Type::FlushTLB,
.flush_tlb = { .flush_tlb = {
.vaddr = vaddr, .vaddr = vaddr,
.page_count = 1, .page_count = pages,
.page_table = vaddr < KERNEL_OFFSET ? this : nullptr, .page_table = vaddr < KERNEL_OFFSET ? this : nullptr,
} }
}); });
} }
} }
void PageTable::unmap_page(vaddr_t vaddr, bool send_smp_message) void PageTable::unmap_page(vaddr_t vaddr, bool invalidate)
{ {
ASSERT(vaddr); ASSERT(vaddr);
ASSERT(vaddr != fast_page()); ASSERT(vaddr != fast_page());
@@ -663,31 +692,23 @@ namespace Kernel
pt[pte] = 0; pt[pte] = 0;
if (old_paddr != 0) if (invalidate && old_paddr != 0)
invalidate(vaddr, send_smp_message); invalidate_page(vaddr, true);
} }
void PageTable::unmap_range(vaddr_t vaddr, size_t size) void PageTable::unmap_range(vaddr_t vaddr, size_t size)
{ {
ASSERT(vaddr % PAGE_SIZE == 0); ASSERT(vaddr % PAGE_SIZE == 0);
size_t page_count = range_page_count(vaddr, size); const size_t page_count = range_page_count(vaddr, size);
SpinLockGuard _(m_lock); SpinLockGuard _(m_lock);
for (vaddr_t page = 0; page < page_count; page++) for (vaddr_t page = 0; page < page_count; page++)
unmap_page(vaddr + page * PAGE_SIZE, false); unmap_page(vaddr + page * PAGE_SIZE, false);
invalidate_range(vaddr, page_count, true);
Processor::broadcast_smp_message({
.type = Processor::SMPMessage::Type::FlushTLB,
.flush_tlb = {
.vaddr = vaddr,
.page_count = page_count,
.page_table = vaddr < KERNEL_OFFSET ? this : nullptr,
}
});
} }
void PageTable::map_page_at(paddr_t paddr, vaddr_t vaddr, flags_t flags, MemoryType memory_type, bool send_smp_message) void PageTable::map_page_at(paddr_t paddr, vaddr_t vaddr, flags_t flags, MemoryType memory_type, bool invalidate)
{ {
ASSERT(vaddr); ASSERT(vaddr);
ASSERT(vaddr != fast_page()); ASSERT(vaddr != fast_page());
@@ -752,8 +773,8 @@ namespace Kernel
pt[pte] = paddr | uwr_flags | extra_flags; pt[pte] = paddr | uwr_flags | extra_flags;
if (old_paddr != 0) if (invalidate && old_paddr != 0)
invalidate(vaddr, send_smp_message); invalidate_page(vaddr, true);
} }
void PageTable::map_range_at(paddr_t paddr, vaddr_t vaddr, size_t size, flags_t flags, MemoryType memory_type) void PageTable::map_range_at(paddr_t paddr, vaddr_t vaddr, size_t size, flags_t flags, MemoryType memory_type)
@@ -769,15 +790,7 @@ namespace Kernel
SpinLockGuard _(m_lock); SpinLockGuard _(m_lock);
for (size_t page = 0; page < page_count; page++) for (size_t page = 0; page < page_count; page++)
map_page_at(paddr + page * PAGE_SIZE, vaddr + page * PAGE_SIZE, flags, memory_type, false); map_page_at(paddr + page * PAGE_SIZE, vaddr + page * PAGE_SIZE, flags, memory_type, false);
invalidate_range(vaddr, page_count, true);
Processor::broadcast_smp_message({
.type = Processor::SMPMessage::Type::FlushTLB,
.flush_tlb = {
.vaddr = vaddr,
.page_count = page_count,
.page_table = vaddr < KERNEL_OFFSET ? this : nullptr,
}
});
} }
uint64_t PageTable::get_page_data(vaddr_t vaddr) const uint64_t PageTable::get_page_data(vaddr_t vaddr) const
@@ -824,13 +837,13 @@ namespace Kernel
return page_data & s_page_addr_mask; return page_data & s_page_addr_mask;
} }
bool PageTable::reserve_page(vaddr_t vaddr, bool only_free, bool send_smp_message) bool PageTable::reserve_page(vaddr_t vaddr, bool only_free, bool invalidate)
{ {
SpinLockGuard _(m_lock); SpinLockGuard _(m_lock);
ASSERT(vaddr % PAGE_SIZE == 0); ASSERT(vaddr % PAGE_SIZE == 0);
if (only_free && !is_page_free(vaddr)) if (only_free && !is_page_free(vaddr))
return false; return false;
map_page_at(0, vaddr, Flags::Reserved, MemoryType::Normal, send_smp_message); map_page_at(0, vaddr, Flags::Reserved, MemoryType::Normal, invalidate);
return true; return true;
} }
@@ -845,14 +858,7 @@ namespace Kernel
return false; return false;
for (size_t offset = 0; offset < bytes; offset += PAGE_SIZE) for (size_t offset = 0; offset < bytes; offset += PAGE_SIZE)
reserve_page(vaddr + offset, true, false); reserve_page(vaddr + offset, true, false);
Processor::broadcast_smp_message({ invalidate_range(vaddr, bytes / PAGE_SIZE, true);
.type = Processor::SMPMessage::Type::FlushTLB,
.flush_tlb = {
.vaddr = vaddr,
.page_count = bytes / PAGE_SIZE,
.page_table = vaddr < KERNEL_OFFSET ? this : nullptr,
}
});
return true; return true;
} }
@@ -868,7 +874,7 @@ namespace Kernel
ASSERT(is_canonical(first_address)); ASSERT(is_canonical(first_address));
ASSERT(is_canonical(last_address - 1)); ASSERT(is_canonical(last_address - 1));
const vaddr_t uc_vaddr_start = uncanonicalize(first_address); const vaddr_t uc_vaddr_start = uncanonicalize(first_address);
const vaddr_t uc_vaddr_end = uncanonicalize(last_address); const vaddr_t uc_vaddr_end = uncanonicalize(last_address - 1);
uint16_t pml4e = (uc_vaddr_start >> 39) & 0x1FF; uint16_t pml4e = (uc_vaddr_start >> 39) & 0x1FF;
uint16_t pdpte = (uc_vaddr_start >> 30) & 0x1FF; uint16_t pdpte = (uc_vaddr_start >> 30) & 0x1FF;
@@ -885,10 +891,8 @@ namespace Kernel
// Try to find free page that can be mapped without // Try to find free page that can be mapped without
// allocations (page table with unused entries) // allocations (page table with unused entries)
const uint64_t* pml4 = P2V(m_highest_paging_struct); const uint64_t* pml4 = P2V(m_highest_paging_struct);
for (; pml4e < 512; pml4e++) for (; pml4e <= e_pml4e; pml4e++)
{ {
if (pml4e > e_pml4e)
break;
if (!(pml4[pml4e] & Flags::Present)) if (!(pml4[pml4e] & Flags::Present))
continue; continue;
const uint64_t* pdpt = P2V(pml4[pml4e] & s_page_addr_mask); const uint64_t* pdpt = P2V(pml4[pml4e] & s_page_addr_mask);
@@ -908,10 +912,10 @@ namespace Kernel
const uint64_t* pt = P2V(pd[pde] & s_page_addr_mask); const uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
for (; pte < 512; pte++) for (; pte < 512; pte++)
{ {
if (pml4e == e_pml4e && pdpte == e_pdpte && pde == e_pde && pte >= e_pte) if (pml4e == e_pml4e && pdpte == e_pdpte && pde == e_pde && pte > e_pte)
break; break;
if (!(pt[pte] & Flags::Used)) if (pt[pte] & Flags::Used)
{ continue;
vaddr_t vaddr = 0; vaddr_t vaddr = 0;
vaddr |= static_cast<uint64_t>(pml4e) << 39; vaddr |= static_cast<uint64_t>(pml4e) << 39;
vaddr |= static_cast<uint64_t>(pdpte) << 30; vaddr |= static_cast<uint64_t>(pdpte) << 30;
@@ -921,9 +925,11 @@ namespace Kernel
ASSERT(reserve_page(vaddr)); ASSERT(reserve_page(vaddr));
return vaddr; return vaddr;
} }
pte = 0;
} }
pde = 0;
} }
} pdpte = 0;
} }
for (vaddr_t uc_vaddr = uc_vaddr_start; uc_vaddr < uc_vaddr_end; uc_vaddr += PAGE_SIZE) for (vaddr_t uc_vaddr = uc_vaddr_start; uc_vaddr < uc_vaddr_end; uc_vaddr += PAGE_SIZE)

11
kernel/include/kernel/Memory/PageTable.h
View File

@@ -100,10 +100,10 @@ namespace Kernel
static BAN::ErrorOr<PageTable*> create_userspace(); static BAN::ErrorOr<PageTable*> create_userspace();
~PageTable(); ~PageTable();
void unmap_page(vaddr_t, bool send_smp_message = true); void unmap_page(vaddr_t, bool invalidate = true);
void unmap_range(vaddr_t, size_t bytes); void unmap_range(vaddr_t, size_t bytes);
void map_page_at(paddr_t, vaddr_t, flags_t, MemoryType = MemoryType::Normal, bool send_smp_message = true); void map_page_at(paddr_t, vaddr_t, flags_t, MemoryType = MemoryType::Normal, bool invalidate = true);
void map_range_at(paddr_t, vaddr_t, size_t bytes, flags_t, MemoryType = MemoryType::Normal); void map_range_at(paddr_t, vaddr_t, size_t bytes, flags_t, MemoryType = MemoryType::Normal);
paddr_t physical_address_of(vaddr_t) const; paddr_t physical_address_of(vaddr_t) const;
@@ -112,7 +112,7 @@ namespace Kernel
bool is_page_free(vaddr_t) const; bool is_page_free(vaddr_t) const;
bool is_range_free(vaddr_t, size_t bytes) const; bool is_range_free(vaddr_t, size_t bytes) const;
bool reserve_page(vaddr_t, bool only_free = true, bool send_smp_message = true); bool reserve_page(vaddr_t, bool only_free = true, bool invalidate = true);
bool reserve_range(vaddr_t, size_t bytes, bool only_free = true); bool reserve_range(vaddr_t, size_t bytes, bool only_free = true);
vaddr_t reserve_free_page(vaddr_t first_address, vaddr_t last_address = UINTPTR_MAX); vaddr_t reserve_free_page(vaddr_t first_address, vaddr_t last_address = UINTPTR_MAX);
@@ -121,6 +121,9 @@ namespace Kernel
void load(); void load();
void initial_load(); void initial_load();
void invalidate_page(vaddr_t addr, bool send_smp_message) { invalidate_range(addr, 1, send_smp_message); }
void invalidate_range(vaddr_t addr, size_t pages, bool send_smp_message);
InterruptState lock() const { return m_lock.lock(); } InterruptState lock() const { return m_lock.lock(); }
void unlock(InterruptState state) const { m_lock.unlock(state); } void unlock(InterruptState state) const { m_lock.unlock(state); }
@@ -133,8 +136,6 @@ namespace Kernel
void map_kernel_memory(); void map_kernel_memory();
void prepare_fast_page(); void prepare_fast_page();
void invalidate(vaddr_t, bool send_smp_message);
static void map_fast_page(paddr_t); static void map_fast_page(paddr_t);
static void unmap_fast_page(); static void unmap_fast_page();

3
kernel/kernel/Processor.cpp
View File

@@ -377,8 +377,7 @@ namespace Kernel
case SMPMessage::Type::FlushTLB: case SMPMessage::Type::FlushTLB:
if (message->flush_tlb.page_table && message->flush_tlb.page_table != processor.m_current_page_table) if (message->flush_tlb.page_table && message->flush_tlb.page_table != processor.m_current_page_table)
break; break;
for (size_t i = 0; i < message->flush_tlb.page_count; i++) PageTable::current().invalidate_range(message->flush_tlb.vaddr, message->flush_tlb.page_count, false);
asm volatile("invlpg (%0)" :: "r"(message->flush_tlb.vaddr + i * PAGE_SIZE) : "memory");
break; break;
case SMPMessage::Type::NewThread: case SMPMessage::Type::NewThread:
processor.m_scheduler->add_thread(message->new_thread); processor.m_scheduler->add_thread(message->new_thread);