Kernel: Processes and Threads use VirtualRange memory allocations

kernel/include/kernel/Process.h

@@ -8,6 +8,7 @@
 #include <kernel/Memory/GeneralAllocator.h>
 #include <kernel/Memory/Heap.h>
 #include <kernel/Memory/MMU.h>
+#include <kernel/Memory/VirtualRange.h>
 #include <kernel/SpinLock.h>
 #include <kernel/Terminal/TTY.h>
 #include <kernel/Thread.h>
@@ -91,7 +92,7 @@ namespace Kernel
 		BAN::ErrorOr<int> get_free_fd();
 
 		BAN::Vector<OpenFileDescription> m_open_files;
-		BAN::Vector<paddr_t> m_allocated_pages;
+		BAN::Vector<VirtualRange*> m_mapped_ranges;
 
 		mutable RecursiveSpinLock m_lock;
 
@@ -99,7 +100,7 @@ namespace Kernel
 		BAN::String m_working_directory;
 		BAN::Vector<Thread*> m_threads;
 
-		BAN::LinkedList<FixedWidthAllocator> m_fixed_width_allocators;
+		BAN::Vector<FixedWidthAllocator*> m_fixed_width_allocators;
 		GeneralAllocator* m_general_allocator;
 
 		MMU* m_mmu { nullptr };

kernel/include/kernel/Thread.h

@@ -2,7 +2,7 @@
 
 #include <BAN/NoCopyMove.h>
 #include <BAN/RefPtr.h>
-#include <kernel/Memory/Heap.h>
+#include <kernel/Memory/VirtualRange.h>
 
 #include <sys/types.h>
 
@@ -27,7 +27,7 @@ namespace Kernel
 		};
 
 	public:
-		static BAN::ErrorOr<Thread*> create(entry_t, void*, Process*);
+		static BAN::ErrorOr<Thread*> create_kernel(entry_t, void*, Process*);
 		static BAN::ErrorOr<Thread*> create_userspace(uintptr_t, Process*, int, char**);
 		~Thread();
 
@@ -42,11 +42,11 @@ namespace Kernel
 		State state() const { return m_state; }
 		void terminate() { m_state = State::Terminating; }
 
-		uintptr_t stack_base() const { return (uintptr_t)m_stack_base; }
+		vaddr_t stack_base() const { return m_stack->vaddr(); }
-		size_t stack_size() const { return m_is_userspace ? m_userspace_stack_size : m_kernel_stack_size; }
+		size_t stack_size() const { return m_stack->size(); }
 
-		uintptr_t interrupt_stack_base() const { return (uintptr_t)m_interrupt_stack; }
+		vaddr_t interrupt_stack_base() const { return m_interrupt_stack ? m_interrupt_stack->vaddr() : 0; }
-		uintptr_t interrupt_stack_size() const { return m_interrupt_stack_size; }
+		size_t interrupt_stack_size() const { return m_interrupt_stack ? m_interrupt_stack->size() : 0; }
 
 		static Thread& current() ;
 		static pid_t current_tid();
@@ -72,10 +72,10 @@ namespace Kernel
 
 	private:
 		static constexpr size_t m_kernel_stack_size		= PAGE_SIZE * 1;
-		static constexpr size_t m_userspace_stack_size	= PAGE_SIZE * 1;
+		static constexpr size_t m_userspace_stack_size	= PAGE_SIZE * 2;
-		static constexpr size_t m_interrupt_stack_size	= PAGE_SIZE;
+		static constexpr size_t m_interrupt_stack_size	= PAGE_SIZE * 2;
-		vaddr_t		m_interrupt_stack	{ 0 };
+		VirtualRange*	m_interrupt_stack	{ nullptr };
-		vaddr_t		m_stack_base		{ 0 };
+		VirtualRange*	m_stack				{ nullptr };
 		uintptr_t		m_rip				{ 0 };
 		uintptr_t		m_rsp				{ 0 };
 		const pid_t		m_tid				{ 0 };

kernel/kernel/Process.cpp

@@ -40,7 +40,7 @@ namespace Kernel
 	{
 		auto* process = create_process();
 		MUST(process->m_working_directory.push_back('/'));
-		auto* thread = MUST(Thread::create(entry, data, process));
+		auto* thread = MUST(Thread::create_kernel(entry, data, process));
 		process->add_thread(thread);
 		register_process(process);
 		return process;
@@ -74,19 +74,14 @@ namespace Kernel
 			{
 				// TODO: Do some relocations or map kernel to higher half?
 				ASSERT(process->mmu().is_range_free(elf_program_header.p_vaddr, elf_program_header.p_memsz));
-
 				MMU::flags_t flags = MMU::Flags::UserSupervisor | MMU::Flags::Present;
 				if (elf_program_header.p_flags & LibELF::PF_W)
 					flags |= MMU::Flags::ReadWrite;
 				size_t page_start = elf_program_header.p_vaddr / PAGE_SIZE;
 				size_t page_end = BAN::Math::div_round_up<size_t>(elf_program_header.p_vaddr + elf_program_header.p_memsz, PAGE_SIZE);
-				MUST(process->m_allocated_pages.reserve(page_end - page_start + 1));
+
-				for (size_t page = page_start; page <= page_end; page++)
+				size_t page_count = page_end - page_start + 1;
-				{
+				MUST(process->m_mapped_ranges.push_back(VirtualRange::create(process->mmu(), page_start * PAGE_SIZE, page_count * PAGE_SIZE, flags)));
-					auto paddr = Heap::get().take_free_page();
-					MUST(process->m_allocated_pages.push_back(paddr));
-					process->mmu().map_page_at(paddr, page * PAGE_SIZE, flags);
-				}
 
 				{
 					MMUScope _(process->mmu());
@@ -128,13 +123,12 @@ namespace Kernel
 		ASSERT(m_threads.empty());
 		ASSERT(m_fixed_width_allocators.empty());
 		ASSERT(m_general_allocator == nullptr);
+		ASSERT(m_mapped_ranges.empty());
 		if (m_mmu)
 		{
 			MMU::kernel().load();
 			delete m_mmu;
 		}
-		for (auto paddr : m_allocated_pages)
-			Heap::get().release_page(paddr);
 	}
 
 	void Process::add_thread(Thread* thread)
@@ -160,6 +154,11 @@ namespace Kernel
 		for (auto& open_fd : m_open_files)
 			open_fd.inode = nullptr;
 
+		// NOTE: We must unmap ranges while the mmu is still alive
+		for (auto* range : m_mapped_ranges)
+			delete range;
+		m_mapped_ranges.clear();
+
 		// NOTE: We must clear allocators while the mmu is still alive
 		m_fixed_width_allocators.clear();
 		if (m_general_allocator)
@@ -169,6 +168,7 @@ namespace Kernel
 		}
 
 		dprintln("process {} exit", pid());
+
 		s_process_lock.lock();
 		for (size_t i = 0; i < s_processes.size(); i++)
 			if (s_processes[i] == this)
@@ -443,19 +443,22 @@ namespace Kernel
 
 			bool needs_new_allocator { true };
 
-			for (auto& allocator : m_fixed_width_allocators)
+			for (auto* allocator : m_fixed_width_allocators)
 			{
-				if (allocator.allocation_size() == allocation_size && allocator.allocations() < allocator.max_allocations())
+				if (allocator->allocation_size() == allocation_size && allocator->allocations() < allocator->max_allocations())
 				{
-					address = allocator.allocate();
+					address = allocator->allocate();
 					needs_new_allocator = false;
 				}
 			}
 
 			if (needs_new_allocator)
 			{
-				TRY(m_fixed_width_allocators.emplace_back(mmu(), allocation_size));
+				auto* allocator = new FixedWidthAllocator(mmu(), allocation_size);
-				address = m_fixed_width_allocators.back().allocate();
+				if (allocator == nullptr)
+					return BAN::Error::from_errno(ENOMEM);
+				TRY(m_fixed_width_allocators.push_back(allocator));
+				address = m_fixed_width_allocators.back()->allocate();
 			}
 		}
 		else
@@ -481,14 +484,15 @@ namespace Kernel
 	{
 		LockGuard _(m_lock);
 
-		for (auto it = m_fixed_width_allocators.begin(); it != m_fixed_width_allocators.end(); it++)
+		for (size_t i = 0; i < m_fixed_width_allocators.size(); i++)
 		{
-			if (it->deallocate((vaddr_t)ptr))
+			auto* allocator = m_fixed_width_allocators[i];
+			if (allocator->deallocate((vaddr_t)ptr))
 			{
 				// TODO: This might be too much. Maybe we should only
 				//       remove allocators when we have low memory... ?
-				if (it->allocations() == 0)
+				if (allocator->allocations() == 0)
-					m_fixed_width_allocators.remove(it);
+					m_fixed_width_allocators.remove(i);
 				return;
 			}
 		}

kernel/kernel/Scheduler.cpp

@@ -23,12 +23,18 @@ namespace Kernel
 
 	static Scheduler* s_instance = nullptr;
 
+	static uint8_t s_temp_stack[1024];
+	ALWAYS_INLINE static void load_temp_stack()
+	{
+		asm volatile("movq %0, %%rsp" :: "r"(s_temp_stack + sizeof(s_temp_stack)));
+	}
+
 	BAN::ErrorOr<void> Scheduler::initialize()
 	{
 		ASSERT(s_instance == nullptr);
 		s_instance = new Scheduler();
 		ASSERT(s_instance);
-		s_instance->m_idle_thread = TRY(Thread::create([](void*) { for (;;) asm volatile("hlt"); }, nullptr, nullptr));
+		s_instance->m_idle_thread = TRY(Thread::create_kernel([](void*) { for (;;) asm volatile("hlt"); }, nullptr, nullptr));
 		return {};
 	}
 
@@ -111,7 +117,7 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Scheduler::add_thread(Thread* thread)
 	{
-		Kernel::CriticalScope critical;
+		CriticalScope _;
 		TRY(m_active_threads.emplace_back(thread));
 		return {};
 	}
@@ -167,6 +173,8 @@ namespace Kernel
 		current.set_rip(rip);
 		current.set_rsp(rsp);
 
+		load_temp_stack();
+
 		return false;
 	}
 
@@ -234,6 +242,8 @@ namespace Kernel
 		VERIFY_STI();
 		DISABLE_INTERRUPTS();
 
+		load_temp_stack();
+
 		ASSERT(m_current_thread);
 
 		Thread* thread = m_current_thread->thread;
@@ -262,14 +272,16 @@ namespace Kernel
 	{
 		DISABLE_INTERRUPTS();
 
-		pid_t pid = m_current_thread->thread->process().pid();
+		load_temp_stack();
 
-		remove_threads(m_blocking_threads, it->thread->process().pid() == pid);
+		Process* process = &m_current_thread->thread->process();
-		remove_threads(m_sleeping_threads, it->thread->process().pid() == pid);
+
-		remove_threads(m_active_threads, it != m_current_thread && it->thread->process().pid() == pid);
+		remove_threads(m_blocking_threads, it->thread->process().pid() == process->pid());
+		remove_threads(m_sleeping_threads, it->thread->process().pid() == process->pid());
+		remove_threads(m_active_threads, it != m_current_thread && it->thread->process().pid() == process->pid());
 
-		delete &m_current_thread->thread->process();
 		delete m_current_thread->thread;
+		delete process;
 		remove_and_advance_current_thread();
 		execute_current_thread();
 

kernel/kernel/Shell.cpp

@@ -399,7 +399,7 @@ argument_done:
 			thread_data_t thread_data = { this, spinlock, arguments };
 			spinlock.lock();
 
-			auto* thread = TRY(Thread::create(function, &thread_data, &Process::current()));
+			auto* thread = TRY(Thread::create_kernel(function, &thread_data, &Process::current()));
 			Process::current().add_thread(thread);
 			while (spinlock.is_locked());
 		}

kernel/kernel/Thread.cpp

@@ -1,7 +1,7 @@
 #include <BAN/Errors.h>
-#include <kernel/CriticalScope.h>
 #include <kernel/InterruptController.h>
 #include <kernel/Memory/kmalloc.h>
+#include <kernel/Memory/MMUScope.h>
 #include <kernel/Process.h>
 #include <kernel/Scheduler.h>
 #include <kernel/Thread.h>
@@ -10,6 +10,7 @@ namespace Kernel
 {
 
 	extern "C" void thread_userspace_trampoline(uint64_t rsp, uint64_t rip, int argc, char** argv);
+	extern "C" uintptr_t read_rip();
 
 	template<size_t size, typename T>
 	static void write_to_stack(uintptr_t& rsp, const T& value)
@@ -20,7 +21,7 @@ namespace Kernel
 
 	static pid_t s_next_tid = 1;
 
-	BAN::ErrorOr<Thread*> Thread::create(entry_t entry, void* data, Process* process)
+	BAN::ErrorOr<Thread*> Thread::create_kernel(entry_t entry, void* data, Process* process)
 	{
 		// Create the thread object
 		Thread* thread = new Thread(s_next_tid++, process);
@@ -28,10 +29,10 @@ namespace Kernel
 			return BAN::Error::from_errno(ENOMEM);
 
 		// Initialize stack and registers
-		thread->m_stack_base = (vaddr_t)kmalloc(m_kernel_stack_size, PAGE_SIZE);
+		thread->m_stack = VirtualRange::create_kmalloc(m_kernel_stack_size);
-		if (thread->m_stack_base == 0)
+		if (thread->m_stack == nullptr)
 			return BAN::Error::from_errno(ENOMEM);
-		thread->m_rsp = (uintptr_t)thread->m_stack_base + m_kernel_stack_size;
+		thread->m_rsp = thread->stack_base() + thread->stack_size();
 		thread->m_rip = (uintptr_t)entry;
 
 		// Initialize stack for returning
@@ -44,6 +45,8 @@ namespace Kernel
 
 	BAN::ErrorOr<Thread*> Thread::create_userspace(uintptr_t entry, Process* process, int argc, char** argv)
 	{
+		ASSERT(process);
+
 		// Create the thread object
 		Thread* thread = new Thread(s_next_tid++, process);
 		if (thread == nullptr)
@@ -51,13 +54,20 @@ namespace Kernel
 		thread->m_is_userspace = true;
 
 		// Allocate stack
-		thread->m_stack_base = (uintptr_t)kmalloc(m_userspace_stack_size, PAGE_SIZE);
+		thread->m_stack = VirtualRange::create(process->mmu(), 0, m_userspace_stack_size, MMU::Flags::UserSupervisor | MMU::Flags::ReadWrite | MMU::Flags::Present);
-		ASSERT(thread->m_stack_base);
+		if (thread->m_stack == nullptr)
-		process->mmu().identity_map_range(thread->m_stack_base, m_userspace_stack_size, MMU::Flags::UserSupervisor | MMU::Flags::ReadWrite | MMU::Flags::Present);
+		{
+			delete thread;
+			return BAN::Error::from_errno(ENOMEM);
+		}
 
 		// Allocate interrupt stack
-		thread->m_interrupt_stack = (vaddr_t)kmalloc(m_interrupt_stack_size, PAGE_SIZE);
+		thread->m_interrupt_stack = VirtualRange::create(process->mmu(), 0, m_interrupt_stack_size, MMU::Flags::UserSupervisor | MMU::Flags::ReadWrite | MMU::Flags::Present);
-		ASSERT(thread->m_interrupt_stack);
+		if (thread->m_interrupt_stack == nullptr)
+		{
+			delete thread;
+			return BAN::Error::from_errno(ENOMEM);
+		}
 
 		thread->m_userspace_entry = { .entry = entry, .argc = argc, .argv = argv };
 
@@ -70,13 +80,16 @@ namespace Kernel
 				ASSERT_NOT_REACHED();
 			}
 		);
-		thread->m_rsp = thread->m_stack_base + m_userspace_stack_size;
+		thread->m_rsp = thread->stack_base() + thread->stack_size();
 		thread->m_rip = (uintptr_t)entry_trampoline;
 
 		// Setup stack for returning
+		{
+			MMUScope _(process->mmu());
 			write_to_stack<sizeof(void*)>(thread->m_rsp, thread);
 			write_to_stack<sizeof(void*)>(thread->m_rsp, &Thread::on_exit);
 			write_to_stack<sizeof(void*)>(thread->m_rsp, nullptr);
+		}
 
 		return thread;
 	}
@@ -99,10 +112,12 @@ namespace Kernel
 	Thread::~Thread()
 	{
 		dprintln("thread {} ({}) exit", tid(), m_process->pid());
-
+		if (m_stack)
+			delete m_stack;
+		m_stack = nullptr;
 		if (m_interrupt_stack)
-			kfree((void*)m_interrupt_stack);
+			delete m_interrupt_stack;
-		kfree((void*)m_stack_base);
+		m_interrupt_stack = nullptr;
 	}
 
 	void Thread::validate_stack() const