Bananymous
/
banan-os
1
1
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Kernel: Rewrite SMP message code 

Remove locks and map smp buffer as uncached
This commit is contained in:
Bananymous 2025-08-27 22:56:30 +03:00
parent 6976a2dae7
commit a8bb07052e
2 changed files with 111 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
kernel/include/kernel/Processor.h
View File

@ -120,6 +120,8 @@ namespace Kernel
static ProcessorID read_processor_id(); static ProcessorID read_processor_id();
static void initialize_smp();
template<typename T> template<typename T>
static T read_gs_sized(uintptr_t offset) requires(sizeof(T) <= 8) static T read_gs_sized(uintptr_t offset) requires(sizeof(T) <= 8)
{ {
@ -173,12 +175,8 @@ namespace Kernel
uint64_t m_last_update_ns { 0 }; uint64_t m_last_update_ns { 0 };
uint64_t m_next_update_ns { 0 }; uint64_t m_next_update_ns { 0 };
BAN::Atomic<bool> m_smp_pending_lock { false }; BAN::Atomic<SMPMessage*> m_smp_pending { nullptr };
SMPMessage* m_smp_pending { nullptr }; BAN::Atomic<SMPMessage*> m_smp_free { nullptr };
BAN::Atomic<bool> m_smp_free_lock { false };
SMPMessage* m_smp_free { nullptr };
SMPMessage* m_smp_message_storage { nullptr }; SMPMessage* m_smp_message_storage { nullptr };
void* m_current_page_table { nullptr }; void* m_current_page_table { nullptr };

204
kernel/kernel/Processor.cpp
View File

@ -1,4 +1,5 @@
#include <kernel/InterruptController.h> #include <kernel/InterruptController.h>
#include <kernel/Memory/Heap.h>
#include <kernel/Memory/kmalloc.h> #include <kernel/Memory/kmalloc.h>
#include <kernel/Processor.h> #include <kernel/Processor.h>
#include <kernel/Terminal/TerminalDriver.h> #include <kernel/Terminal/TerminalDriver.h>
@ -68,15 +69,6 @@ namespace Kernel
processor.m_scheduler = MUST(Scheduler::create()); processor.m_scheduler = MUST(Scheduler::create());
ASSERT(processor.m_scheduler); ASSERT(processor.m_scheduler);
SMPMessage* smp_storage = new SMPMessage[0x1000];
ASSERT(smp_storage);
for (size_t i = 0; i < 0xFFF; i++)
smp_storage[i].next = &smp_storage[i + 1];
smp_storage[0xFFF].next = nullptr;
processor.m_smp_pending = nullptr;
processor.m_smp_free = smp_storage;
s_processors_created++; s_processors_created++;
return processor; return processor;
@ -107,6 +99,34 @@ namespace Kernel
return processor; return processor;
} }
void Processor::initialize_smp()
{
const auto processor_id = current_id();
auto& processor = s_processors[processor_id.as_u32()];
const paddr_t smp_paddr = Heap::get().take_free_page();
ASSERT(smp_paddr);
const vaddr_t smp_vaddr = PageTable::kernel().reserve_free_page(KERNEL_OFFSET);
ASSERT(smp_vaddr);
PageTable::kernel().map_page_at(
smp_paddr, smp_vaddr,
PageTable::Flags::ReadWrite | PageTable::Flags::Present,
PageTable::MemoryType::Uncached
);
auto* smp_storage = reinterpret_cast<SMPMessage*>(smp_vaddr);
constexpr size_t smp_storage_entries = PAGE_SIZE / sizeof(SMPMessage);
for (size_t i = 0; i < smp_storage_entries - 1; i++)
smp_storage[i].next = &smp_storage[i + 1];
smp_storage[smp_storage_entries - 1].next = nullptr;
processor.m_smp_pending = nullptr;
processor.m_smp_free = smp_storage;
}
ProcessorID Processor::id_from_index(size_t index) ProcessorID Processor::id_from_index(size_t index)
{ {
ASSERT(index < s_processor_count); ASSERT(index < s_processor_count);
@ -116,12 +136,7 @@ namespace Kernel
void Processor::wait_until_processors_ready() void Processor::wait_until_processors_ready()
{ {
if (s_processors_created == 1) initialize_smp();
{
ASSERT(current_is_bsp());
s_processor_count++;
s_processor_ids[0] = current_id();
}
// wait until bsp is ready // wait until bsp is ready
if (current_is_bsp()) if (current_is_bsp())
@ -182,21 +197,6 @@ namespace Kernel
handle_smp_messages(); handle_smp_messages();
} }
template<typename F>
void with_atomic_lock(BAN::Atomic<bool>& lock, F callback)
{
bool expected = false;
while (!lock.compare_exchange(expected, true, BAN::MemoryOrder::memory_order_acquire))
{
__builtin_ia32_pause();
expected = false;
}
callback();
lock.store(false, BAN::MemoryOrder::memory_order_release);
}
void Processor::handle_smp_messages() void Processor::handle_smp_messages()
{ {
auto state = get_interrupt_state(); auto state = get_interrupt_state();
@ -205,65 +205,61 @@ namespace Kernel
auto processor_id = current_id(); auto processor_id = current_id();
auto& processor = s_processors[processor_id.m_id]; auto& processor = s_processors[processor_id.m_id];
SMPMessage* pending = nullptr; auto* pending = processor.m_smp_pending.exchange(nullptr);
with_atomic_lock(processor.m_smp_pending_lock, if (pending == nullptr)
[&]() return set_interrupt_state(state);
{
pending = processor.m_smp_pending;
processor.m_smp_pending = nullptr;
}
);
if (pending) // reverse smp message queue from LIFO to FIFO
{ {
// reverse smp message queue from LIFO to FIFO SMPMessage* reversed = nullptr;
for (auto* message = pending; message;)
{ {
SMPMessage* reversed = nullptr; SMPMessage* next = message->next;
message->next = reversed;
for (SMPMessage* message = pending; message;) reversed = message;
{ message = next;
SMPMessage* next = message->next;
message->next = reversed;
reversed = message;
message = next;
}
pending = reversed;
} }
SMPMessage* last_handled = nullptr; pending = reversed;
}
// handle messages SMPMessage* last_handled = nullptr;
for (auto* message = pending; message; message = message->next)
// handle messages
for (auto* message = pending; message; message = message->next)
{
switch (message->type)
{ {
switch (message->type) case SMPMessage::Type::FlushTLB:
{ for (size_t i = 0; i < message->flush_tlb.page_count; i++)
case SMPMessage::Type::FlushTLB: asm volatile("invlpg (%0)" :: "r"(message->flush_tlb.vaddr + i * PAGE_SIZE) : "memory");
for (size_t i = 0; i < message->flush_tlb.page_count; i++) break;
asm volatile("invlpg (%0)" :: "r"(message->flush_tlb.vaddr + i * PAGE_SIZE) : "memory"); case SMPMessage::Type::NewThread:
break; processor.m_scheduler->add_thread(message->new_thread);
case SMPMessage::Type::NewThread: break;
processor.m_scheduler->add_thread(message->new_thread); case SMPMessage::Type::UnblockThread:
break; processor.m_scheduler->unblock_thread(message->unblock_thread);
case SMPMessage::Type::UnblockThread: break;
processor.m_scheduler->unblock_thread(message->unblock_thread); #if WITH_PROFILING
break; case SMPMessage::Type::StartProfiling:
case SMPMessage::Type::StackTrace: processor.start_profiling();
dwarnln("Stack trace of CPU {}", current_id().as_u32()); break;
Debug::dump_stack_trace(); #endif
break; case SMPMessage::Type::StackTrace:
} dwarnln("Stack trace of CPU {}", current_id().as_u32());
Debug::dump_stack_trace();
last_handled = message; break;
} }
with_atomic_lock(processor.m_smp_free_lock, last_handled = message;
[&]() }
{
last_handled->next = processor.m_smp_free; last_handled->next = processor.m_smp_free;
processor.m_smp_free = pending; while (!processor.m_smp_free.compare_exchange(last_handled->next, pending))
} {
); __builtin_ia32_pause();
last_handled->next = processor.m_smp_free;
} }
set_interrupt_state(state); set_interrupt_state(state);
@ -283,35 +279,49 @@ namespace Kernel
void Processor::send_smp_message(ProcessorID processor_id, const SMPMessage& message, bool send_ipi) void Processor::send_smp_message(ProcessorID processor_id, const SMPMessage& message, bool send_ipi)
{ {
auto state = get_interrupt_state(); auto state = get_interrupt_state();
set_interrupt_state(InterruptState::Disabled); set_interrupt_state(InterruptState::Disabled);
auto& processor = s_processors[processor_id.m_id]; auto& processor = s_processors[processor_id.m_id];
// take free message slot // find a slot for message
SMPMessage* storage = nullptr; auto* storage = processor.m_smp_free.exchange(nullptr);
with_atomic_lock(processor.m_smp_free_lock, while (storage == nullptr)
[&]() {
{ __builtin_ia32_pause();
storage = processor.m_smp_free; storage = processor.m_smp_free.exchange(nullptr);
ASSERT(storage && storage->next); }
processor.m_smp_free = storage->next; if (auto* base = storage->next)
{
SMPMessage* null = nullptr;
if (!processor.m_smp_free.compare_exchange(null, base))
{
// NOTE: this is an annoying traversal, but most of the time
// above if condition bypasses this :)
auto* last = base;
while (last->next)
last = last->next;
last->next = processor.m_smp_free;
while (!processor.m_smp_free.compare_exchange(last->next, base))
{
__builtin_ia32_pause();
last->next = processor.m_smp_free;
}
} }
); }
// write message // write message
*storage = message; *storage = message;
// push message to pending queue // push message to pending queue
with_atomic_lock(processor.m_smp_pending_lock, storage->next = processor.m_smp_pending;
[&]() while (!processor.m_smp_pending.compare_exchange(storage->next, storage))
{ {
storage->next = processor.m_smp_pending; __builtin_ia32_pause();
processor.m_smp_pending = storage; storage->next = processor.m_smp_pending;
} }
);
if (send_ipi) if (send_ipi)
{ {