Kernel: Per processor information is now stored in class Processor

This allows us to allocate processor stacks, and other per processor
structures dynamically in runtime. Giving processor stack to
ap_trampoline feels super hacky, but it works for now.

kernel/CMakeLists.txt

@@ -66,6 +66,7 @@ set(KERNEL_SOURCES
 	kernel/PCI.cpp
 	kernel/PIC.cpp
 	kernel/Process.cpp
+	kernel/Processor.cpp
 	kernel/Random.cpp
 	kernel/Scheduler.cpp
 	kernel/Semaphore.cpp

kernel/arch/x86_64/boot.S

@@ -53,9 +53,9 @@ bananboot_start:
 bananboot_end:
 
 .section .bss, "aw", @nobits
-	# reserve 4096 bytes of initial stack for each processor
+	boot_stack_bottom:
-	g_processor_stacks:
+		.skip 4096 * 4
-		.skip 4096 * 64
+	boot_stack_top:
 
 	.global g_kernel_cmdline
 	g_kernel_cmdline:
@@ -108,6 +108,9 @@ g_ap_startup_done:
 .global g_ap_running_count
 g_ap_running_count:
 	.byte 0
+.global g_ap_stack_loaded
+g_ap_stack_loaded:
+	.byte 0
 
 .section .text
 
@@ -155,19 +158,12 @@ enable_sse:
 	movl %eax, %cr4
 	ret
 
-# NOTE: return address in argument %edi
+initialize_lapic_id:
-initialize_pmode_stack:
 	movl $1, %eax
 	cpuid
 	shrl $24, %ebx
-
 	movw %bx, %gs
-
+	ret
-	shll $12, %ebx
-	addl $V2P(g_processor_stacks) + 4096, %ebx
-	movl %ebx, %esp
-
-	jmp *%edi
 
 initialize_paging:
 	# enable PAE
@@ -195,13 +191,14 @@ initialize_paging:
 .global _start
 .type _start, @function
 _start:
+	cli; cld
+
 	# Initialize stack and multiboot info
 	movl %eax, V2P(bootloader_magic)
 	movl %ebx, V2P(bootloader_info)
 
-	movl $V2P(1f), %edi
+	movl $V2P(boot_stack_top), %esp
-	jmp initialize_pmode_stack
+	call initialize_lapic_id
-1:
 
 	call check_requirements
 	call enable_sse
@@ -253,7 +250,12 @@ system_halt:
 .code16
 .global ap_trampoline
 ap_trampoline:
-	cli
+	jmp 1f
+.align 8
+ap_stack_ptr:
+	.skip 4
+1:
+	cli; cld
 	ljmpl $0x00, $ap_cs_clear
 
 ap_cs_clear:
@@ -274,12 +276,15 @@ ap_protected_mode:
 	movw %ax, %ss
 	movw %ax, %es
 
-	movl $1f, %edi
+	movl ap_stack_ptr, %esp
-	jmp V2P(initialize_pmode_stack)
+	movb $1, V2P(g_ap_stack_loaded)
-1:
+	call V2P(initialize_lapic_id)
+
+	call V2P(enable_sse)
 
-	# load boot gdt, load boot page table and enter long mode
 	call V2P(initialize_paging)
+
+	# load boot gdt and enter long mode
 	lgdt V2P(boot_gdtr)
 	ljmpl $0x08, $ap_long_mode
 

kernel/include/kernel/Lock/SpinLock.h

@@ -50,10 +50,10 @@ namespace Kernel
 
 		InterruptState lock()
 		{
-			auto id = get_processor_id();
+			auto id = Processor::current_id();
 
-			auto state = get_interrupt_state();
+			auto state = Processor::get_interrupt_state();
-			set_interrupt_state(InterruptState::Disabled);
+			Processor::set_interrupt_state(InterruptState::Disabled);
 
 			while (!m_locker.compare_exchange(PROCESSOR_NONE, id, BAN::MemoryOrder::memory_order_acquire))
 				__builtin_ia32_pause();
@@ -64,7 +64,7 @@ namespace Kernel
 		void unlock(InterruptState state)
 		{
 			m_locker.store(PROCESSOR_NONE, BAN::MemoryOrder::memory_order_release);
-			set_interrupt_state(state);
+			Processor::set_interrupt_state(state);
 		}
 
 		bool is_locked() const { return m_locker != PROCESSOR_NONE; }

kernel/include/kernel/Processor.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <BAN/ForwardList.h>
 #include <kernel/Arch.h>
 
 namespace Kernel
@@ -11,9 +12,29 @@ namespace Kernel
 		Enabled,
 	};
 
-#if ARCH(x86_64) || ARCH(i386)
+	using ProcessorID = uint8_t;
+	constexpr ProcessorID PROCESSOR_NONE = 0xFF;
 
-	inline void set_interrupt_state(InterruptState state)
+#if ARCH(x86_64)
+	class Processor
+	{
+		BAN_NON_COPYABLE(Processor);
+
+	public:
+		static Processor& create(ProcessorID id);
+
+		static ProcessorID current_id()
+		{
+			uint16_t id;
+			asm volatile("movw %%gs, %0" : "=rm"(id));
+			return id;
+		}
+
+		static Processor& get(ProcessorID);
+
+		static Processor& current() { return get(current_id()); }
+
+		static void set_interrupt_state(InterruptState state)
 		{
 			if (state == InterruptState::Enabled)
 				asm volatile("sti");
@@ -21,26 +42,35 @@ namespace Kernel
 				asm volatile("cli");
 		}
 
-	inline InterruptState get_interrupt_state()
+		static InterruptState get_interrupt_state()
 		{
 			uintptr_t flags;
 			asm volatile("pushf; pop %0" : "=rm"(flags));
 			if (flags & (1 << 9))
 				return InterruptState::Enabled;
 			return InterruptState::Disabled;
-	}
+		};
 
-	using ProcessorID = uint8_t;
+		uintptr_t stack_bottom() const { return reinterpret_cast<uintptr_t>(m_stack); }
-	constexpr ProcessorID PROCESSOR_NONE = 0xFF;
+		uintptr_t stack_top() const { return stack_bottom() + m_stack_size; }
-	inline ProcessorID get_processor_id()
+
+	private:
+		Processor() = default;
+		Processor(Processor&& other)
 		{
-		uint16_t id;
+			m_stack = other.m_stack;
-		asm volatile("movw %%gs, %0" : "=rm"(id));
+			other.m_stack = nullptr;
-		return id;
 		}
+		~Processor();
 
+	private:
+		void* m_stack { nullptr };
+		static constexpr size_t m_stack_size { 4096 };
+
+		friend class BAN::Vector<Processor>;
+	};
 #else
-#error "Unknown architecure"
+	#error
 #endif
 
 }

kernel/kernel/APIC.cpp

@@ -26,6 +26,7 @@ extern uint8_t g_ap_init_addr[];
 
 extern volatile uint8_t g_ap_startup_done[];
 extern volatile uint8_t g_ap_running_count[];
+extern volatile uint8_t g_ap_stack_loaded[];
 
 namespace Kernel
 {
@@ -215,7 +216,7 @@ namespace Kernel
 
 		dprintln("System has {} processors", m_processors.size());
 
-		uint8_t bsp_id = get_processor_id();
+		uint8_t bsp_id = Kernel::Processor::current_id();
 		dprintln("BSP lapic id: {}", bsp_id);
 
 		for (auto& processor : m_processors)
@@ -231,6 +232,13 @@ namespace Kernel
 
 			dprintln("Trying to enable processor (lapic id {})", processor.apic_id);
 
+			Kernel::Processor::create(processor.processor_id);
+
+			PageTable::with_fast_page((paddr_t)g_ap_init_addr, [&] {
+				PageTable::fast_page_as_sized<uint32_t>(2) = V2P(Kernel::Processor::get(processor.processor_id).stack_top());
+			});
+			*g_ap_stack_loaded = 0;
+
 			write_to_local_apic(LAPIC_ERROR_REG, 0x00);
 			send_ipi(processor.processor_id, (read_from_local_apic(LAPIC_ICR_LO_REG) & 0xFFF00000) | 0x0000C500, 0);
 			send_ipi(processor.processor_id, (read_from_local_apic(LAPIC_ICR_LO_REG) & 0xFFF0F800) | 0x00008500, 0);
@@ -242,12 +250,13 @@ namespace Kernel
 				write_to_local_apic(LAPIC_ERROR_REG, 0x00);
 				send_ipi(processor.processor_id, (read_from_local_apic(LAPIC_ICR_LO_REG) & 0xFFF0F800) | 0x00000600 | ap_init_page, 200);
 			}
+
+			// give processor upto 100 * 100 us (10 ms to boot)
+			for (int i = 0; *g_ap_stack_loaded == 0 && i < 100; i++)
+				udelay(100);
 		}
 
 		*g_ap_startup_done = 1;
-
-		udelay(100);
-
 		dprintln("{} processors started", *g_ap_running_count);
 	}
 

kernel/kernel/Lock/SpinLock.cpp

@@ -9,11 +9,11 @@ namespace Kernel
 
 	InterruptState SpinLock::lock()
 	{
-		auto id = get_processor_id();
+		auto id = Processor::current_id();
 		ASSERT_NEQ(m_locker.load(), id);
 
-		auto state = get_interrupt_state();
+		auto state = Processor::get_interrupt_state();
-		set_interrupt_state(InterruptState::Disabled);
+		Processor::set_interrupt_state(InterruptState::Disabled);
 
 		while (!m_locker.compare_exchange(PROCESSOR_NONE, id, BAN::MemoryOrder::memory_order_acquire))
 			__builtin_ia32_pause();
@@ -23,17 +23,17 @@ namespace Kernel
 
 	void SpinLock::unlock(InterruptState state)
 	{
-		ASSERT_EQ(m_locker.load(), get_processor_id());
+		ASSERT_EQ(m_locker.load(), Processor::current_id());
 		m_locker.store(PROCESSOR_NONE, BAN::MemoryOrder::memory_order_release);
-		set_interrupt_state(state);
+		Processor::set_interrupt_state(state);
 	}
 
 	InterruptState RecursiveSpinLock::lock()
 	{
-		auto id = get_processor_id();
+		auto id = Processor::current_id();
 
-		auto state = get_interrupt_state();
+		auto state = Processor::get_interrupt_state();
-		set_interrupt_state(InterruptState::Disabled);
+		Processor::set_interrupt_state(InterruptState::Disabled);
 
 		if (id == m_locker)
 			ASSERT_GT(m_lock_depth, 0);
@@ -51,11 +51,11 @@ namespace Kernel
 
 	void RecursiveSpinLock::unlock(InterruptState state)
 	{
-		ASSERT_EQ(m_locker.load(), get_processor_id());
+		ASSERT_EQ(m_locker.load(), Processor::current_id());
 		ASSERT_GT(m_lock_depth, 0);
 		if (--m_lock_depth == 0)
 			m_locker.store(PROCESSOR_NONE, BAN::MemoryOrder::memory_order_release);
-		set_interrupt_state(state);
+		Processor::set_interrupt_state(state);
 	}
 
 }

kernel/kernel/Memory/kmalloc.cpp

@@ -75,6 +75,11 @@ struct kmalloc_info
 		return nullptr;
 	}
 
+	bool contains(uintptr_t addr) const
+	{
+		return base <= addr && addr < end;
+	}
+
 	size_t used = 0;
 	size_t free = size;
 };
@@ -161,8 +166,17 @@ static bool is_corrupted()
 	Kernel::SpinLockGuard _(s_kmalloc_lock);
 	auto& info = s_kmalloc_info;
 	auto* temp = info.first();
-	for (; temp->end() <= info.end; temp = temp->after());
+	while (reinterpret_cast<uintptr_t>(temp) != info.end)
-	return (uintptr_t)temp != info.end;
+	{
+		if (!info.contains(reinterpret_cast<uintptr_t>(temp)))
+			return true;
+		if (!info.contains(temp->end() - 1))
+			return true;
+		if (temp->after() <= temp)
+			return true;
+		temp = temp->after();
+	}
+	return false;
 }
 
 [[maybe_unused]] static void debug_dump()
@@ -235,12 +249,12 @@ static void* kmalloc_impl(size_t size, size_t align)
 
 	auto& info = s_kmalloc_info;
 
-	for (auto* node = info.first(); node->end() <= info.end; node = node->after())
+	for (auto* node = info.first(); info.contains(reinterpret_cast<uintptr_t>(node)); node = node->after())
 	{
 		if (node->used())
 			continue;
 
-		if (auto* next = node->after(); next->end() <= info.end)
+		if (auto* next = node->after(); info.contains(reinterpret_cast<uintptr_t>(next)))
 			if (!next->used())
 				node->set_end(next->end());
 

kernel/kernel/Process.cpp

@@ -220,7 +220,7 @@ namespace Kernel
 
 	void Process::on_thread_exit(Thread& thread)
 	{
-		ASSERT(get_interrupt_state() == InterruptState::Disabled);
+		ASSERT(Processor::get_interrupt_state() == InterruptState::Disabled);
 
 		ASSERT(m_threads.size() > 0);
 

kernel/kernel/Processor.cpp

@@ -0,0 +1,34 @@
+#include <BAN/Vector.h>
+#include <kernel/Processor.h>
+
+namespace Kernel
+{
+
+	static BAN::Vector<Processor> s_processors;
+
+	Processor& Processor::create(ProcessorID id)
+	{
+		while (s_processors.size() <= id)
+			MUST(s_processors.emplace_back());
+		auto& processor = s_processors[id];
+		if (processor.m_stack == nullptr)
+		{
+			processor.m_stack = kmalloc(m_stack_size, 4096, true);
+			ASSERT(processor.m_stack);
+		}
+		return processor;
+	}
+
+	Processor::~Processor()
+	{
+		if (m_stack)
+			kfree(m_stack);
+		m_stack = nullptr;
+	}
+
+	Processor& Processor::get(ProcessorID id)
+	{
+		return s_processors[id];
+	}
+
+}

kernel/kernel/Scheduler.cpp

@@ -16,10 +16,9 @@ namespace Kernel
 
 	static Scheduler* s_instance = nullptr;
 
-	static uint8_t s_temp_stack[1024];
 	ALWAYS_INLINE static void load_temp_stack()
 	{
-		asm volatile("movq %0, %%rsp" :: "r"(s_temp_stack + sizeof(s_temp_stack)));
+		asm volatile("movq %0, %%rsp" :: "rm"(Processor::current().stack_top()));
 	}
 
 	BAN::ErrorOr<void> Scheduler::initialize()
@@ -40,7 +39,7 @@ namespace Kernel
 
 	void Scheduler::start()
 	{
-		ASSERT(get_interrupt_state() == InterruptState::Disabled);
+		ASSERT(Processor::get_interrupt_state() == InterruptState::Disabled);
 		m_lock.lock();
 		ASSERT(!m_active_threads.empty());
 		m_current_thread = m_active_threads.begin();
@@ -75,7 +74,7 @@ namespace Kernel
 	{
 		auto state = m_lock.lock();
 		if (save_current_thread())
-			return set_interrupt_state(state);
+			return Processor::set_interrupt_state(state);
 		advance_current_thread();
 		execute_current_thread_locked();
 		ASSERT_NOT_REACHED();
@@ -210,6 +209,7 @@ namespace Kernel
 
 	void Scheduler::execute_current_thread_locked()
 	{
+		ASSERT(m_lock.is_locked());
 		load_temp_stack();
 		PageTable::kernel().load();
 		execute_current_thread_stack_loaded();
@@ -220,13 +220,10 @@ namespace Kernel
 	{
 		ASSERT(m_lock.is_locked());
 
-		load_temp_stack();
-		PageTable::kernel().load();
-
 #if SCHEDULER_VERIFY_STACK
 		vaddr_t rsp;
 		read_rsp(rsp);
-		ASSERT((vaddr_t)s_temp_stack <= rsp && rsp <= (vaddr_t)s_temp_stack + sizeof(s_temp_stack));
+		ASSERT(Processor::current().stack_bottom() <= rsp && rsp <= Processor::current().stack_top());
 		ASSERT(&PageTable::current() == &PageTable::kernel());
 #endif
 
@@ -287,10 +284,7 @@ namespace Kernel
 		ASSERT(m_lock.is_locked());
 
 		if (save_current_thread())
-		{
-			set_interrupt_state(InterruptState::Enabled);
 			return;
-		}
 
 		auto it = m_sleeping_threads.begin();
 		for (; it != m_sleeping_threads.end(); it++)

kernel/kernel/kernel.cpp

@@ -83,8 +83,6 @@ extern "C" void kernel_main(uint32_t boot_magic, uint32_t boot_info)
 {
 	using namespace Kernel;
 
-	set_interrupt_state(InterruptState::Disabled);
-
 	if (!validate_boot_magic(boot_magic))
 	{
 		Serial::initialize();
@@ -104,6 +102,9 @@ extern "C" void kernel_main(uint32_t boot_magic, uint32_t boot_info)
 	parse_boot_info(boot_magic, boot_info);
 	dprintln("boot info parsed");
 
+	Processor::create(Processor::current_id());
+	dprintln("BSP initialized");
+
 	GDT::initialize();
 	dprintln("GDT initialized");
 
@@ -214,7 +215,7 @@ extern "C" void ap_main()
 {
 	using namespace Kernel;
 
-	dprintln("hello from processor {}", get_processor_id());
+	dprintln("hello from processor {}", Processor::current_id());
 
 	for (;;)
 		asm volatile("");