Kernel: Reduce ext2 locking

Replace the mutex with a rwlock and lock when its not necessary

BAN/include/BAN/Sort.h

@@ -21,26 +21,47 @@ namespace BAN::sort
 	namespace detail
 	{
 
-		template<typename It, typename Comp>
-		It partition(It begin, It end, Comp comp)
+		template<typename It>
+		struct partition_pair
 		{
-			It pivot = prev(end, 1);
+			It lt;
+			It gt;
+		};
 
-			It it1 = begin;
-			for (It it2 = begin; it2 != pivot; ++it2)
+		template<typename It, typename Comp>
+		partition_pair<It> partition(It begin, It end, Comp comp)
+		{
+			It pivot = next(begin, distance(begin, end) / 2);
+
+			It lt = begin;
+			It eq = begin;
+			It gt = end;
+
+			while (eq != gt)
 			{
-				if (comp(*it2, *pivot))
+				if (comp(*eq, *pivot))
 				{
-					swap(*it1, *it2);
-					++it1;
+					swap(*eq, *lt);
+					if (pivot == lt)
+						pivot = eq;
+					++lt;
+					++eq;
+				}
+				else if (comp(*pivot, *eq))
+				{
+					--gt;
+					swap(*eq, *gt);
+					if (pivot == gt)
+						pivot = eq;
+				}
+				else
+				{
+					++eq;
 				}
 			}
 
-			swap(*it1, *pivot);
-
-			return it1;
+			return { lt, gt };
 		}
-
 	}
 
 	template<typename It, typename Comp = less<it_value_type_t<It>>>
@@ -48,9 +69,9 @@ namespace BAN::sort
 	{
 		if (distance(begin, end) <= 1)
 			return;
-		It mid = detail::partition(begin, end, comp);
-		quick_sort(begin, mid, comp);
-		quick_sort(++mid, end, comp);
+		const auto [lt, gt] = detail::partition(begin, end, comp);
+		quick_sort(begin, lt, comp);
+		quick_sort(gt, end, comp);
 	}
 
 	template<typename It, typename Comp = less<it_value_type_t<It>>>
@@ -85,9 +106,9 @@ namespace BAN::sort
 				return insertion_sort(begin, end, comp);
 			if (max_depth == 0)
 				return heap_sort(begin, end, comp);
-			It mid = detail::partition(begin, end, comp);
-			intro_sort_impl(begin, mid, max_depth - 1, comp);
-			intro_sort_impl(++mid, end, max_depth - 1, comp);
+			const auto [lt, gt] = detail::partition(begin, end, comp);
+			intro_sort_impl(begin, lt, max_depth - 1, comp);
+			intro_sort_impl(gt, end, max_depth - 1, comp);
 		}
 
 	}
@@ -116,27 +137,20 @@ namespace BAN::sort
 
 	template<typename It, size_t radix = 256>
 	requires is_unsigned_v<it_value_type_t<It>> && (radix > 0 && (radix & (radix - 1)) == 0)
-	BAN::ErrorOr<void> radix_sort(It begin, It end)
+	void radix_sort(It begin, It end, BAN::Span<it_value_type_t<It>> storage)
 	{
-		using value_type = it_value_type_t<It>;
-
 		const size_t len = distance(begin, end);
 		if (len <= 1)
-			return {};
+			return;
 
-		Vector<value_type> temp;
-		TRY(temp.resize(len));
-
-		Vector<size_t> counts;
-		TRY(counts.resize(radix));
+		ASSERT(storage.size() >= len);
 
 		constexpr size_t mask  = radix - 1;
 		constexpr size_t shift = detail::lsb_index(radix);
 
-		for (size_t s = 0; s < sizeof(value_type) * 8; s += shift)
+		for (size_t s = 0; s < sizeof(it_value_type_t<It>) * 8; s += shift)
 		{
-			for (auto& cnt : counts)
-				cnt = 0;
+			size_t counts[radix] {};
 			for (It it = begin; it != end; ++it)
 				counts[(*it >> s) & mask]++;
 
@@ -146,12 +160,27 @@ namespace BAN::sort
 			for (It it = end; it != begin;)
 			{
 				--it;
-				temp[--counts[(*it >> s) & mask]] = *it;
+				storage[--counts[(*it >> s) & mask]] = *it;
 			}
 
-			for (size_t j = 0; j < temp.size(); j++)
-				*next(begin, j) = temp[j];
+			It it = begin;
+			for (size_t j = 0; j < storage.size(); j++, ++it)
+				*it = storage[j];
 		}
+	}
+
+	template<typename It, size_t radix = 256>
+	requires is_unsigned_v<it_value_type_t<It>> && (radix > 0 && (radix & (radix - 1)) == 0)
+	BAN::ErrorOr<void> radix_sort(It begin, It end)
+	{
+		const size_t len = distance(begin, end);
+		if (len <= 1)
+			return {};
+
+		Vector<it_value_type_t<It>> temp;
+		TRY(temp.resize(len));
+
+		radix_sort(begin, end, temp.span());
 
 		return {};
 	}

kernel/arch/i686/PageTable.cpp

@@ -1,7 +1,7 @@
 #include <kernel/BootInfo.h>
 #include <kernel/CPUID.h>
 #include <kernel/Lock/SpinLock.h>
-#include <kernel/Memory/kmalloc.h>
+#include <kernel/Memory/Heap.h>
 #include <kernel/Memory/PageTable.h>
 
 extern uint8_t g_kernel_start[];
@@ -37,24 +37,42 @@ namespace Kernel
 
 	static uint64_t* s_fast_page_pt { nullptr };
 
-	static uint64_t* allocate_zeroed_page_aligned_page()
+	alignas(PAGE_SIZE) static uint64_t s_fast_page_pt_storage[512] {};
+
+	static paddr_t allocate_zeroed_page_aligned_page()
 	{
-		void* page = kmalloc(PAGE_SIZE, PAGE_SIZE, true);
-		ASSERT(page);
-		memset(page, 0, PAGE_SIZE);
-		return (uint64_t*)page;
+		const paddr_t paddr = Heap::get().take_free_page();
+		ASSERT(paddr);
+
+		PageTable::with_fast_page(paddr, [] {
+			memset(PageTable::fast_page_as_ptr(), 0, PAGE_SIZE);
+		});
+
+		return paddr;
 	}
 
-	template<typename T>
-	static paddr_t V2P(const T vaddr)
+	static void unallocate_page(paddr_t paddr)
 	{
-		return (vaddr_t)vaddr - KERNEL_OFFSET + g_boot_info.kernel_paddr;
+		Heap::get().release_page(paddr);
 	}
 
-	template<typename T>
-	static uint64_t* P2V(const T paddr)
+	static uint64_t read_entry_from_table(paddr_t paddr, uint16_t entry)
 	{
-		return reinterpret_cast<uint64_t*>(reinterpret_cast<paddr_t>(paddr) - g_boot_info.kernel_paddr + KERNEL_OFFSET);
+		uint64_t result;
+		PageTable::with_fast_page(paddr & s_page_addr_mask, [&result, entry] {
+			result = PageTable::fast_page_as_sized<uint64_t>(entry);
+		});
+		return result;
+	}
+
+	static uint64_t write_entry_to_table(paddr_t paddr, uint16_t entry, uint64_t value)
+	{
+		uint64_t old_value;
+		PageTable::with_fast_page(paddr & s_page_addr_mask, [&old_value, entry, value] {
+			old_value = PageTable::fast_page_as_sized<uint64_t>(entry);
+			PageTable::fast_page_as_sized<uint64_t>(entry) = value;
+		});
+		return old_value;
 	}
 
 	static inline PageTable::flags_t parse_flags(uint64_t entry)
@@ -142,15 +160,10 @@ namespace Kernel
 		s_kernel = new PageTable();
 		ASSERT(s_kernel);
 
-		auto* pdpt = allocate_zeroed_page_aligned_page();
-		ASSERT(pdpt);
-
-		s_kernel->m_highest_paging_struct = V2P(pdpt);
+		s_kernel->m_highest_paging_struct = allocate_zeroed_page_aligned_page();
 		s_kernel->map_kernel_memory();
 
-		PageTable::with_fast_page(s_kernel->m_highest_paging_struct, [] {
-			s_global_pdpte = PageTable::fast_page_as_sized<paddr_t>(3);
-		});
+		s_global_pdpte = read_entry_from_table(s_kernel->m_highest_paging_struct, 3);
 
 		// update fast page pt
 		{
@@ -161,23 +174,21 @@ namespace Kernel
 			const auto get_or_allocate_entry =
 				[](paddr_t table_paddr, uint16_t entry, uint64_t flags)
 				{
-					uint64_t* table = P2V(table_paddr);
+					const uint64_t value = read_entry_from_table(table_paddr, entry);
+					if (value & Flags::Present)
+						return value & s_page_addr_mask;
 
-					if (!(table[entry] & Flags::Present))
-					{
-						auto* vaddr = allocate_zeroed_page_aligned_page();
-						ASSERT(vaddr);
-						table[entry] = V2P(vaddr);
-					}
-
-					table[entry] |= flags;
-
-					return table[entry] & s_page_addr_mask;
+					const paddr_t paddr = allocate_zeroed_page_aligned_page();
+					write_entry_to_table(table_paddr, entry, paddr | flags);
+					return paddr;
 				};
 
 			const paddr_t pdpt = s_kernel->m_highest_paging_struct;
 			const paddr_t pd = get_or_allocate_entry(pdpt, pdpte, Flags::Present);
-			s_fast_page_pt = P2V(get_or_allocate_entry(pd, pde, Flags::ReadWrite | Flags::Present));
+
+			const paddr_t entry_paddr = reinterpret_cast<uintptr_t>(&s_fast_page_pt_storage) - KERNEL_OFFSET + g_boot_info.kernel_paddr;
+			write_entry_to_table(pd, pde, entry_paddr | PageTable::Flags::ReadWrite | PageTable::Flags::Present);
+			s_fast_page_pt = s_fast_page_pt_storage;
 		}
 
 		s_kernel->load();
@@ -197,33 +208,37 @@ namespace Kernel
 	void PageTable::map_kernel_memory()
 	{
 		// Map (phys_kernel_start -> phys_kernel_end) to (virt_kernel_start -> virt_kernel_end)
+		const vaddr_t kernel_start = reinterpret_cast<vaddr_t>(g_kernel_start);
 		map_range_at(
-			V2P(g_kernel_start),
-			reinterpret_cast<vaddr_t>(g_kernel_start),
+			kernel_start - KERNEL_OFFSET + g_boot_info.kernel_paddr,
+			kernel_start,
 			g_kernel_end - g_kernel_start,
 			Flags::Present
 		);
 
 		// Map executable kernel memory as executable
+		const vaddr_t kernel_execute_start = reinterpret_cast<vaddr_t>(g_kernel_execute_start);
 		map_range_at(
-			V2P(g_kernel_execute_start),
-			reinterpret_cast<vaddr_t>(g_kernel_execute_start),
+			kernel_execute_start - KERNEL_OFFSET + g_boot_info.kernel_paddr,
+			kernel_execute_start,
 			g_kernel_execute_end - g_kernel_execute_start,
 			Flags::Execute | Flags::Present
 		);
 
 		// Map writable kernel memory as writable
+		const vaddr_t kernel_writable_start = reinterpret_cast<vaddr_t>(g_kernel_writable_start);
 		map_range_at(
-			V2P(g_kernel_writable_start),
-			reinterpret_cast<vaddr_t>(g_kernel_writable_start),
+			kernel_writable_start - KERNEL_OFFSET + g_boot_info.kernel_paddr,
+			kernel_writable_start,
 			g_kernel_writable_end - g_kernel_writable_start,
 			Flags::ReadWrite | Flags::Present
 		);
 
 		// Map userspace memory
+		const vaddr_t kernel_userspace_start = reinterpret_cast<vaddr_t>(g_userspace_start);
 		map_range_at(
-			V2P(g_userspace_start),
-			reinterpret_cast<vaddr_t>(g_userspace_start),
+			kernel_userspace_start - KERNEL_OFFSET + g_boot_info.kernel_paddr,
+			kernel_userspace_start,
 			g_userspace_end - g_userspace_start,
 			Flags::Execute | Flags::UserSupervisor | Flags::Present
 		);
@@ -231,26 +246,40 @@ namespace Kernel
 
 	void PageTable::map_fast_page(paddr_t paddr)
 	{
-		ASSERT(paddr && paddr % PAGE_SIZE == 0);
-
-		ASSERT(s_fast_page_pt);
-		ASSERT(s_fast_page_lock.current_processor_has_lock());
-
-		ASSERT(!(*s_fast_page_pt & Flags::Present));
-		s_fast_page_pt[0] = paddr | Flags::ReadWrite | Flags::Present;
-
-		asm volatile("invlpg (%0)" :: "r"(fast_page()));
+		map_fast_page(0, paddr);
 	}
 
 	void PageTable::unmap_fast_page()
 	{
+		unmap_fast_page(0);
+	}
+
+	void* PageTable::map_fast_page(size_t index, paddr_t paddr)
+	{
+		ASSERT(paddr && paddr % PAGE_SIZE == 0);
+
+		ASSERT(index < 512);
 		ASSERT(s_fast_page_pt);
 		ASSERT(s_fast_page_lock.current_processor_has_lock());
 
-		ASSERT((*s_fast_page_pt & Flags::Present));
-		s_fast_page_pt[0] = 0;
+		ASSERT(!(s_fast_page_pt[index] & Flags::Present));
+		s_fast_page_pt[index] = paddr | Flags::ReadWrite | Flags::Present;
 
-		asm volatile("invlpg (%0)" :: "r"(fast_page()));
+		void* address = reinterpret_cast<void*>(fast_page() + index * PAGE_SIZE);
+		asm volatile("invlpg (%0)" :: "r"(address));
+		return address;
+	}
+
+	void PageTable::unmap_fast_page(size_t index)
+	{
+		ASSERT(index < 512);
+		ASSERT(s_fast_page_pt);
+		ASSERT(s_fast_page_lock.current_processor_has_lock());
+
+		ASSERT((s_fast_page_pt[index] & Flags::Present));
+		s_fast_page_pt[index] = 0;
+
+		asm volatile("invlpg (%0)" :: "r"(fast_page() + index * PAGE_SIZE));
 	}
 
 	BAN::ErrorOr<PageTable*> PageTable::create_userspace()
@@ -260,20 +289,16 @@ namespace Kernel
 		if (page_table == nullptr)
 			return BAN::Error::from_errno(ENOMEM);
 
-		uint64_t* pdpt = allocate_zeroed_page_aligned_page();
-		if (pdpt == nullptr)
-		{
-			delete page_table;
-			return BAN::Error::from_errno(ENOMEM);
-		}
+		page_table->m_highest_paging_struct = allocate_zeroed_page_aligned_page();
 
-		page_table->m_highest_paging_struct = V2P(pdpt);
-
-		pdpt[0] = 0;
-		pdpt[1] = 0;
-		pdpt[2] = 0;
-		pdpt[3] = s_global_pdpte | Flags::Present;
-		static_assert(KERNEL_OFFSET == 0xC0000000);
+		PageTable::with_fast_page(page_table->m_highest_paging_struct, [] {
+			uint64_t* pdpt = &PageTable::fast_page_as<uint64_t>();
+			pdpt[0] = 0;
+			pdpt[1] = 0;
+			pdpt[2] = 0;
+			pdpt[3] = s_global_pdpte | Flags::Present;
+			static_assert(KERNEL_OFFSET == 0xC0000000);
+		});
 
 		return page_table;
 	}
@@ -283,21 +308,22 @@ namespace Kernel
 		if (m_highest_paging_struct == 0)
 			return;
 
-		uint64_t* pdpt = P2V(m_highest_paging_struct);
+		const uint64_t pdpt = m_highest_paging_struct;
 		for (uint32_t pdpte = 0; pdpte < 3; pdpte++)
 		{
-			if (!(pdpt[pdpte] & Flags::Present))
+			const uint64_t pd = read_entry_from_table(pdpt, pdpte);
+			if (!(pd & Flags::Present))
 				continue;
-			uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
 			for (uint32_t pde = 0; pde < 512; pde++)
 			{
-				if (!(pd[pde] & Flags::Present))
+				const uint64_t pt = read_entry_from_table(pd, pde);
+				if (!(pt & Flags::Present))
 					continue;
-				kfree(P2V(pd[pde] & s_page_addr_mask));
+				unallocate_page(pt & s_page_addr_mask);
 			}
-			kfree(pd);
+			unallocate_page(pd & s_page_addr_mask);
 		}
-		kfree(pdpt);
+		unallocate_page(m_highest_paging_struct);
 	}
 
 	void PageTable::load()
@@ -317,8 +343,8 @@ namespace Kernel
 			;
 		else if (pages <= 32 || !s_is_initialized)
 		{
-			for (size_t i = 0; i < pages; i++, vaddr += PAGE_SIZE)
-				asm volatile("invlpg (%0)" :: "r"(vaddr));
+			for (size_t i = 0; i < pages; i++)
+				asm volatile("invlpg (%0)" :: "r"(vaddr + i * PAGE_SIZE));
 		}
 		else if (is_userspace || !s_has_pge)
 		{
@@ -374,15 +400,13 @@ namespace Kernel
 		if (is_page_free(vaddr))
 			Kernel::panic("trying to unmap unmapped page 0x{H}", vaddr);
 
-		uint64_t* pdpt = P2V(m_highest_paging_struct);
-		uint64_t* pd   = P2V(pdpt[pdpte] & s_page_addr_mask);
-		uint64_t* pt   = P2V(pd[pde] & s_page_addr_mask);
+		const uint64_t pdpt = m_highest_paging_struct;
+		const uint64_t pd = read_entry_from_table(pdpt, pdpte);
+		const uint64_t pt = read_entry_from_table(pd, pde);
 
-		const paddr_t old_paddr = pt[pte] & s_page_addr_mask;
+		const uint64_t old_entry = write_entry_to_table(pt, pte, 0);
 
-		pt[pte] = 0;
-
-		if (invalidate && old_paddr != 0)
+		if (invalidate && (old_entry & s_page_addr_mask))
 			invalidate_page(vaddr, true);
 	}
 
@@ -433,28 +457,31 @@ namespace Kernel
 
 		SpinLockGuard _(m_lock);
 
-		uint64_t* pdpt = P2V(m_highest_paging_struct);
-		if (!(pdpt[pdpte] & Flags::Present))
-			pdpt[pdpte] = V2P(allocate_zeroed_page_aligned_page()) | Flags::Present;
+		const uint64_t pdpt = m_highest_paging_struct;
 
-		uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
-		if ((pd[pde] & uwr_flags) != uwr_flags)
+		uint64_t pd = read_entry_from_table(pdpt, pdpte);
+		if (!(pd & Flags::Present))
 		{
-			if (!(pd[pde] & Flags::Present))
-				pd[pde] = V2P(allocate_zeroed_page_aligned_page());
-			pd[pde] |= uwr_flags;
+			pd = allocate_zeroed_page_aligned_page();
+			pd |= Flags::Present;
+			write_entry_to_table(pdpt, pdpte, pd);
+		}
+
+		uint64_t pt = read_entry_from_table(pd, pde);
+		if ((pt & uwr_flags) != uwr_flags)
+		{
+			if (!(pt & Flags::Present))
+				pt = allocate_zeroed_page_aligned_page();
+			pt |= uwr_flags;
+			write_entry_to_table(pd, pde, pt);
 		}
 
 		if (!(flags & Flags::Present))
 			uwr_flags &= ~Flags::Present;
 
-		uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
+		const uint64_t old_entry = write_entry_to_table(pt, pte, paddr | uwr_flags | extra_flags);
 
-		const paddr_t old_paddr = pt[pte] & s_page_addr_mask;
-
-		pt[pte] = paddr | uwr_flags | extra_flags;
-
-		if (invalidate && old_paddr != 0)
+		if (invalidate && (old_entry & s_page_addr_mask))
 			invalidate_page(vaddr, true);
 	}
 
@@ -485,31 +512,36 @@ namespace Kernel
 		const uint32_t e_pde   = ((vaddr + size - 1) >> 21) & 0x1FF;
 		const uint32_t e_pte   = ((vaddr + size - 1) >> 12) & 0x1FF;
 
-		SpinLockGuard _(m_lock);
+		SpinLockGuard _0(m_lock);
 
-		const uint64_t* pdpt = P2V(m_highest_paging_struct);
+		SpinLockGuard _1(s_fast_page_lock);
+
+		const uint64_t* pdpt = static_cast<uint64_t*>(map_fast_page(0, m_highest_paging_struct));
 		for (; pdpte <= e_pdpte; pdpte++)
 		{
 			if (!(pdpt[pdpte] & Flags::Present))
 				continue;
-			const uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
+			const uint64_t* pd = static_cast<uint64_t*>(map_fast_page(1, pdpt[pdpte] & s_page_addr_mask));
 			for (; pde < 512; pde++)
 			{
 				if (pdpte == e_pdpte && pde > e_pde)
 					break;
 				if (!(pd[pde] & Flags::ReadWrite))
 					continue;
-				uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
+				uint64_t* pt = static_cast<uint64_t*>(map_fast_page(2, pd[pde] & s_page_addr_mask));
 				for (; pte < 512; pte++)
 				{
 					if (pdpte == e_pdpte && pde == e_pde && pte > e_pte)
 						break;
 					pt[pte] &= ~static_cast<uint64_t>(Flags::ReadWrite);
 				}
+				unmap_fast_page(2);
 				pte = 0;
 			}
+			unmap_fast_page(1);
 			pde = 0;
 		}
+		unmap_fast_page(0);
 
 		invalidate_range(vaddr, size / PAGE_SIZE, true);
 	}
@@ -524,19 +556,17 @@ namespace Kernel
 
 		SpinLockGuard _(m_lock);
 
-		const uint64_t* pdpt = P2V(m_highest_paging_struct);
-		if (!(pdpt[pdpte] & Flags::Present))
+		const uint64_t pdpt = m_highest_paging_struct;
+
+		const uint64_t pd = read_entry_from_table(pdpt, pdpte);
+		if (!(pd & Flags::Present))
 			return 0;
 
-		const uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
-		if (!(pd[pde] & Flags::Present))
+		const uint64_t pt = read_entry_from_table(pd, pde);
+		if (!(pt & Flags::Present))
 			return 0;
 
-		const uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
-		if (!(pt[pte] & Flags::Used))
-			return 0;
-
-		return pt[pte];
+		return read_entry_from_table(pt, pte);
 	}
 
 	PageTable::flags_t PageTable::get_page_flags(vaddr_t vaddr) const
@@ -612,27 +642,35 @@ namespace Kernel
 
 		SpinLockGuard _(m_lock);
 
+		auto state = s_fast_page_lock.lock();
+
 		// Try to find free page that can be mapped without
 		// allocations (page table with unused entries)
-		const uint64_t* pdpt = P2V(m_highest_paging_struct);
+		const uint64_t* pdpt = static_cast<uint64_t*>(map_fast_page(0, m_highest_paging_struct));
 		for (; pdpte <= e_pdpte; pdpte++)
 		{
 			if (!(pdpt[pdpte] & Flags::Present))
 				continue;
-			const uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
+			const uint64_t* pd = static_cast<uint64_t*>(map_fast_page(1, pdpt[pdpte] & s_page_addr_mask));
 			for (; pde < 512; pde++)
 			{
 				if (pdpte == e_pdpte && pde > e_pde)
 					break;
 				if (!(pd[pde] & Flags::Present))
 					continue;
-				const uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
+				const uint64_t* pt = static_cast<uint64_t*>(map_fast_page(2, pd[pde] & s_page_addr_mask));
 				for (; pte < 512; pte++)
 				{
 					if (pdpte == e_pdpte && pde == e_pde && pte > e_pte)
 						break;
 					if (pt[pte] & Flags::Used)
 						continue;
+
+					unmap_fast_page(2);
+					unmap_fast_page(1);
+					unmap_fast_page(0);
+					s_fast_page_lock.unlock(state);
+
 					vaddr_t vaddr = 0;
 					vaddr |= (vaddr_t)pdpte << 30;
 					vaddr |= (vaddr_t)pde   << 21;
@@ -640,10 +678,15 @@ namespace Kernel
 					ASSERT(reserve_page(vaddr));
 					return vaddr;
 				}
+				unmap_fast_page(2);
 				pte = 0;
 			}
+			unmap_fast_page(1);
 			pde = 0;
 		}
+		unmap_fast_page(0);
+
+		s_fast_page_lock.unlock(state);
 
 		// Find any free page
 		for (vaddr_t vaddr = first_address; vaddr < last_address; vaddr += PAGE_SIZE)
@@ -706,12 +749,14 @@ namespace Kernel
 
 	void PageTable::debug_dump()
 	{
-		SpinLockGuard _(m_lock);
+		SpinLockGuard _0(m_lock);
 
 		flags_t flags = 0;
 		vaddr_t start = 0;
 
-		const uint64_t* pdpt = P2V(m_highest_paging_struct);
+		SpinLockGuard _1(s_fast_page_lock);
+
+		const uint64_t* pdpt = static_cast<uint64_t*>(map_fast_page(0, m_highest_paging_struct));
 		for (uint32_t pdpte = 0; pdpte < 4; pdpte++)
 		{
 			if (!(pdpt[pdpte] & Flags::Present))
@@ -720,7 +765,7 @@ namespace Kernel
 				start = 0;
 				continue;
 			}
-			const uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
+			const uint64_t* pd = static_cast<uint64_t*>(map_fast_page(1, pdpt[pdpte] & s_page_addr_mask));
 			for (uint64_t pde = 0; pde < 512; pde++)
 			{
 				if (!(pd[pde] & Flags::Present))
@@ -729,7 +774,7 @@ namespace Kernel
 					start = 0;
 					continue;
 				}
-				const uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
+				const uint64_t* pt = static_cast<uint64_t*>(map_fast_page(2, pd[pde] & s_page_addr_mask));
 				for (uint64_t pte = 0; pte < 512; pte++)
 				{
 					if (parse_flags(pt[pte]) != flags)
@@ -747,8 +792,11 @@ namespace Kernel
 						start = (pdpte << 30) | (pde << 21) | (pte << 12);
 					}
 				}
+				unmap_fast_page(2);
 			}
+			unmap_fast_page(1);
 		}
+		unmap_fast_page(0);
 	}
 
 }

kernel/arch/x86_64/PageTable.cpp

@@ -369,26 +369,40 @@ namespace Kernel
 
 	void PageTable::map_fast_page(paddr_t paddr)
 	{
-		ASSERT(paddr && paddr % PAGE_SIZE == 0);
-
-		ASSERT(s_fast_page_pt);
-		ASSERT(s_fast_page_lock.current_processor_has_lock());
-
-		ASSERT(!(*s_fast_page_pt & Flags::Present));
-		s_fast_page_pt[0] = paddr | Flags::ReadWrite | Flags::Present;
-
-		asm volatile("invlpg (%0)" :: "r"(fast_page()));
+		map_fast_page(0, paddr);
 	}
 
 	void PageTable::unmap_fast_page()
 	{
+		unmap_fast_page(0);
+	}
+
+	void* PageTable::map_fast_page(size_t index, paddr_t paddr)
+	{
+		ASSERT(paddr && paddr % PAGE_SIZE == 0);
+
+		ASSERT(index < 512);
 		ASSERT(s_fast_page_pt);
 		ASSERT(s_fast_page_lock.current_processor_has_lock());
 
-		ASSERT((*s_fast_page_pt & Flags::Present));
-		s_fast_page_pt[0] = 0;
+		ASSERT(!(s_fast_page_pt[index] & Flags::Present));
+		s_fast_page_pt[index] = paddr | Flags::ReadWrite | Flags::Present;
 
-		asm volatile("invlpg (%0)" :: "r"(fast_page()));
+		void* address = reinterpret_cast<void*>(fast_page() + index * PAGE_SIZE);
+		asm volatile("invlpg (%0)" :: "r"(address));
+		return address;
+	}
+
+	void PageTable::unmap_fast_page(size_t index)
+	{
+		ASSERT(index < 512);
+		ASSERT(s_fast_page_pt);
+		ASSERT(s_fast_page_lock.current_processor_has_lock());
+
+		ASSERT((s_fast_page_pt[index] & Flags::Present));
+		s_fast_page_pt[index] = 0;
+
+		asm volatile("invlpg (%0)" :: "r"(fast_page() + index * PAGE_SIZE));
 	}
 
 	BAN::ErrorOr<PageTable*> PageTable::create_userspace()
@@ -458,8 +472,8 @@ namespace Kernel
 			;
 		else if (pages <= 32 || !s_is_initialized)
 		{
-			for (size_t i = 0; i < pages; i++, vaddr += PAGE_SIZE)
-				asm volatile("invlpg (%0)" :: "r"(vaddr));
+			for (size_t i = 0; i < pages; i++)
+				asm volatile("invlpg (%0)" :: "r"(vaddr + i * PAGE_SIZE));
 		}
 		else if (is_userspace || !s_has_pge)
 		{
@@ -729,8 +743,7 @@ namespace Kernel
 
 	paddr_t PageTable::physical_address_of(vaddr_t addr) const
 	{
-		uint64_t page_data = get_page_data(addr);
-		return page_data & s_page_addr_mask;
+		return get_page_data(addr) & s_page_addr_mask;
 	}
 
 	bool PageTable::reserve_page(vaddr_t vaddr, bool only_free, bool invalidate)
@@ -861,7 +874,7 @@ namespace Kernel
 			{
 				if (!is_canonical(vaddr + page * PAGE_SIZE))
 				{
-					vaddr = canonicalize(uncanonicalize(vaddr) + page * PAGE_SIZE);
+					vaddr = canonicalize(uncanonicalize(vaddr + page * PAGE_SIZE));
 					valid = false;
 					break;
 				}

kernel/include/kernel/Epoll.h

@@ -2,6 +2,7 @@
 
 #include <BAN/HashMap.h>
 #include <kernel/FS/Inode.h>
+#include <kernel/Lock/Mutex.h>
 
 #include <sys/epoll.h>
 
@@ -90,6 +91,7 @@ namespace Kernel
 		};
 
 	private:
+		Mutex m_mutex;
 		ThreadBlocker m_thread_blocker;
 		SpinLock m_ready_lock;
 		BAN::HashMap<BAN::RefPtr<Inode>, uint32_t> m_ready_events;

kernel/include/kernel/FS/EventFD.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <kernel/FS/Inode.h>
+#include <kernel/Lock/Mutex.h>
 
 namespace Kernel
 {
@@ -44,8 +45,9 @@ namespace Kernel
 
 	private:
 		const bool m_is_semaphore;
-		uint64_t m_value;
+		BAN::Atomic<uint64_t> m_value;
 
+		Mutex m_mutex;
 		ThreadBlocker m_thread_blocker;
 	};
 

kernel/include/kernel/FS/Ext2/Inode.h

@@ -4,6 +4,7 @@
 #include <BAN/StringView.h>
 #include <kernel/FS/Ext2/Definitions.h>
 #include <kernel/FS/Inode.h>
+#include <kernel/Lock/RWLock.h>
 
 namespace Kernel
 {
@@ -57,25 +58,30 @@ namespace Kernel
 		virtual bool has_hungup_impl() const override { return false; }
 
 	private:
+		uint32_t block_group() const;
+
 		// Returns maximum number of data blocks in use
 		// NOTE: the inode might have more blocks than what this suggests if it has been shrinked
 		uint32_t max_used_data_block_count() const { return size() / blksize(); }
 
-		BAN::ErrorOr<BAN::Optional<uint32_t>> block_from_indirect_block(uint32_t& block, uint32_t index, uint32_t depth, bool allocate);
-		BAN::ErrorOr<BAN::Optional<uint32_t>> fs_block_of_data_block_index(uint32_t data_block_index, bool allocate);
+		BAN::ErrorOr<void> sync_no_lock();
 
-		BAN::ErrorOr<void> link_inode_to_directory(Ext2Inode&, BAN::StringView name);
-		BAN::ErrorOr<void> remove_inode_from_directory(BAN::StringView name, bool cleanup_directory);
-		BAN::ErrorOr<bool> is_directory_empty();
+		BAN::ErrorOr<bool> is_directory_empty_no_lock();
+		BAN::ErrorOr<BAN::RefPtr<Inode>> find_inode_no_lock(BAN::StringView);
 
-		BAN::ErrorOr<void> cleanup_indirect_block(uint32_t block, uint32_t depth);
-		BAN::ErrorOr<void> cleanup_default_links();
-		BAN::ErrorOr<void> cleanup_data_blocks();
-		BAN::ErrorOr<void> cleanup_from_fs();
+		/* needs write end of the lock when allocate is true*/
+		BAN::ErrorOr<BAN::Optional<uint32_t>> block_from_indirect_block_no_lock(uint32_t& block, uint32_t index, uint32_t depth, bool allocate);
+		BAN::ErrorOr<BAN::Optional<uint32_t>> fs_block_of_data_block_index_no_lock(uint32_t data_block_index, bool allocate);
 
-		BAN::ErrorOr<void> sync();
+		/* needs write end of the lock */
+		BAN::ErrorOr<void> link_inode_to_directory_no_lock(Ext2Inode&, BAN::StringView name);
+		BAN::ErrorOr<void> remove_inode_from_directory_no_lock(BAN::StringView name, bool cleanup_directory);
 
-		uint32_t block_group() const;
+		/* needs write end of the lock */
+		BAN::ErrorOr<void> cleanup_indirect_block_no_lock(uint32_t block, uint32_t depth);
+		BAN::ErrorOr<void> cleanup_default_links_no_lock();
+		BAN::ErrorOr<void> cleanup_data_blocks_no_lock();
+		BAN::ErrorOr<void> cleanup_from_fs_no_lock();
 
 	private:
 		Ext2Inode(Ext2FS& fs, Ext2::Inode inode, uint32_t ino)
@@ -97,7 +103,7 @@ namespace Kernel
 			{
 				if (memcmp(&inode.m_inode, &inode_info, sizeof(Ext2::Inode)) == 0)
 					return;
-				if (auto ret = inode.sync(); ret.is_error())
+				if (auto ret = inode.sync_no_lock(); ret.is_error())
 					dwarnln("failed to sync inode: {}", ret.error());
 			}
 
@@ -110,6 +116,8 @@ namespace Kernel
 		Ext2::Inode m_inode;
 		const uint32_t m_ino;
 
+		RWLock m_lock;
+
 		friend class Ext2FS;
 		friend class BAN::RefPtr<Ext2Inode>;
 	};

kernel/include/kernel/FS/Inode.h

@@ -10,7 +10,6 @@
 
 #include <kernel/Credentials.h>
 #include <kernel/Debug.h>
-#include <kernel/Lock/Mutex.h>
 
 #include <dirent.h>
 #include <sys/socket.h>
@@ -183,11 +182,10 @@ namespace Kernel
 
 		virtual BAN::ErrorOr<long> ioctl_impl(int, void*) { return BAN::Error::from_errno(ENOTSUP); }
 
-	protected:
-		mutable PriorityMutex m_mutex;
-
 	private:
+		SpinLock m_shared_region_lock;
 		BAN::WeakPtr<SharedFileData> m_shared_region;
+
 		SpinLock m_epoll_lock;
 		BAN::LinkedList<class Epoll*> m_epolls;
 		friend class Epoll;

kernel/include/kernel/FS/Pipe.h

@@ -1,7 +1,7 @@
 #pragma once
 
-#include <BAN/Array.h>
 #include <kernel/FS/Inode.h>
+#include <kernel/Lock/Mutex.h>
 #include <kernel/Memory/ByteRingBuffer.h>
 #include <kernel/ThreadBlocker.h>
 
@@ -44,6 +44,8 @@ namespace Kernel
 		virtual bool has_error_impl() const override { return m_reading_count == 0; }
 		virtual bool has_hungup_impl() const override { return m_writing_count == 0; }
 
+		virtual BAN::ErrorOr<long> ioctl_impl(int, void*) override;
+
 	private:
 		Pipe(const Credentials&);
 
@@ -53,6 +55,8 @@ namespace Kernel
 		timespec m_atime {};
 		timespec m_mtime {};
 		timespec m_ctime {};
+
+		Mutex m_mutex;
 		ThreadBlocker m_thread_blocker;
 
 		BAN::UniqPtr<ByteRingBuffer> m_buffer;

kernel/include/kernel/FS/TmpFS/Inode.h

@@ -4,6 +4,7 @@
 #include <BAN/Optional.h>
 #include <kernel/FS/Inode.h>
 #include <kernel/FS/TmpFS/Definitions.h>
+#include <kernel/Lock/Mutex.h>
 
 namespace Kernel
 {
@@ -53,22 +54,25 @@ namespace Kernel
 		virtual BAN::ErrorOr<void> fsync_impl() override { return {}; }
 
 		void sync();
-		virtual BAN::ErrorOr<void> prepare_unlink() { return {}; };
+		virtual BAN::ErrorOr<void> prepare_unlink_no_lock() { return {}; };
 
 		void free_all_blocks();
-		void free_indirect_blocks(size_t block, uint32_t depth);
+		void free_indirect_blocks_no_lock(size_t block, uint32_t depth);
 
 		BAN::Optional<size_t> block_index(size_t data_block_index);
-		BAN::Optional<size_t> block_index_from_indirect(size_t block, size_t index, uint32_t depth);
+		BAN::Optional<size_t> block_index_from_indirect_no_lock(size_t block, size_t index, uint32_t depth);
 
 		BAN::ErrorOr<size_t> block_index_with_allocation(size_t data_block_index);
-		BAN::ErrorOr<size_t> block_index_from_indirect_with_allocation(size_t& block, size_t index, uint32_t depth);
+		BAN::ErrorOr<size_t> block_index_from_indirect_with_allocation_no_lock(size_t& block, size_t index, uint32_t depth);
 
 	protected:
 		TmpFileSystem& m_fs;
 		TmpInodeInfo m_inode_info;
 		const ino_t m_ino;
 
+		// TODO: try to reduce locking or replace this with rwlock(?)
+		Mutex m_lock;
+
 		// has to be able to increase link count
 		friend class TmpDirectoryInode;
 	};
@@ -149,7 +153,7 @@ namespace Kernel
 	protected:
 		TmpDirectoryInode(TmpFileSystem&, ino_t, const TmpInodeInfo&);
 
-		virtual BAN::ErrorOr<void> prepare_unlink() override;
+		virtual BAN::ErrorOr<void> prepare_unlink_no_lock() override;
 
 	protected:
 		virtual BAN::ErrorOr<BAN::RefPtr<Inode>> find_inode_impl(BAN::StringView) override final;

kernel/include/kernel/Lock/SpinLockAsMutex.h

@@ -46,7 +46,7 @@ namespace Kernel
 		uint32_t lock_depth() const override { return m_lock_depth; }
 
 	private:
-		SpinLock& m_lock;
+		Lock& m_lock;
 		uint32_t m_lock_depth { 0 };
 		InterruptState m_state;
 		const pid_t m_locker;

kernel/include/kernel/Memory/FileBackedRegion.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <kernel/FS/Inode.h>
+#include <kernel/Lock/RWLock.h>
 #include <kernel/Memory/MemoryRegion.h>
 
 namespace Kernel
@@ -10,15 +11,14 @@ namespace Kernel
 	{
 		~SharedFileData();
 
-		void sync(size_t page_index);
+		void sync_no_lock(size_t page_index);
 
-		Mutex mutex;
+		RWLock rw_lock;
 
 		// FIXME: this should probably be ordered tree like map
 		//        for fast lookup and less memory usage
 		BAN::Vector<paddr_t> pages;
 		BAN::RefPtr<Inode> inode;
-		uint8_t page_buffer[PAGE_SIZE];
 	};
 
 	class FileBackedRegion final : public MemoryRegion

kernel/include/kernel/Memory/PageTable.h

@@ -149,6 +149,9 @@ namespace Kernel
 		static void map_fast_page(paddr_t);
 		static void unmap_fast_page();
 
+		static void* map_fast_page(size_t index, paddr_t);
+		static void unmap_fast_page(size_t index);
+
 	private:
 		paddr_t						m_highest_paging_struct { 0 };
 		mutable RecursiveSpinLock	m_lock;

kernel/include/kernel/Memory/kmalloc.h

@@ -1,15 +1,8 @@
 #pragma once
 
-#include <BAN/Optional.h>
-#include <kernel/Memory/Types.h>
-
 #include <stddef.h>
 
 void kmalloc_initialize();
-void kmalloc_dump_info();
 
-void* kmalloc(size_t size);
-void* kmalloc(size_t size, size_t align, bool force_identity_map = false);
+void* kmalloc(size_t);
 void kfree(void*);
-
-BAN::Optional<Kernel::paddr_t> kmalloc_paddr_of(Kernel::vaddr_t);

kernel/include/kernel/Networking/E1000/E1000.h

@@ -73,11 +73,15 @@ namespace Kernel
 		BAN::UniqPtr<DMARegion>	m_tx_buffer_region;
 		BAN::UniqPtr<DMARegion>	m_rx_descriptor_region;
 		BAN::UniqPtr<DMARegion>	m_tx_descriptor_region;
-		SpinLock m_lock;
+
+		BAN::Atomic<uint32_t> m_tx_head1 { 0 };
+		BAN::Atomic<uint32_t> m_tx_head2 { 0 };
+
+		SpinLock m_rx_lock;
+		ThreadBlocker m_rx_blocker;
 
 		bool m_thread_should_die { false };
 		BAN::Atomic<bool> m_thread_is_dead { true };
-		ThreadBlocker m_thread_blocker;
 
 		BAN::MACAddress	m_mac_address {};
 		bool			m_link_up { false };

kernel/include/kernel/Networking/TCPSocket.h

@@ -181,6 +181,7 @@ namespace Kernel
 
 		uint64_t m_time_wait_start_ms { 0 };
 
+		mutable Mutex m_mutex;
 		ThreadBlocker m_thread_blocker;
 
 		RecvWindowInfo m_recv_window;

kernel/include/kernel/Networking/UDPSocket.h

@@ -66,9 +66,12 @@ namespace Kernel
 		SpinLock							m_packet_lock;
 		ThreadBlocker						m_packet_thread_blocker;
 
+		SpinLock							m_peer_address_lock;
 		sockaddr_storage					m_peer_address {};
 		socklen_t							m_peer_address_len { 0 };
 
+		Mutex								m_bind_lock;
+
 		friend class BAN::RefPtr<UDPSocket>;
 	};
 

kernel/include/kernel/Networking/UNIX/Socket.h

@@ -43,15 +43,16 @@ namespace Kernel
 		UnixDomainSocket(Socket::Type, const Socket::Info&);
 		~UnixDomainSocket();
 
-		bool is_bound() const { return !m_bound_file.canonical_path.empty(); }
-		bool is_bound_to_unused() const { return !m_bound_file.inode; }
+		bool is_bound() const;
+		bool is_bound_to_unused() const;
+		BAN::ErrorOr<void> bind_to_unused_if_not_bound();
 
 		bool is_streaming() const;
 
 	private:
 		struct ConnectionInfo
 		{
-			bool										listening { false };
+			BAN::Atomic<bool>							listening { false };
 			BAN::Atomic<bool>							connection_done { false };
 			mutable BAN::Atomic<bool>					target_closed { false };
 			BAN::WeakPtr<UnixDomainSocket>				connection;
@@ -62,6 +63,7 @@ namespace Kernel
 
 		struct ConnectionlessInfo
 		{
+			SpinLock lock;
 			BAN::String peer_address;
 		};
 
@@ -76,7 +78,9 @@ namespace Kernel
 		BAN::ErrorOr<size_t> add_packet(const msghdr&, PacketInfo&&, bool dont_block);
 
 	private:
-		const Socket::Type		m_socket_type;
+		const Socket::Type m_socket_type;
+
+		mutable Mutex m_bind_mutex;
 		VirtualFileSystem::File	m_bound_file;
 
 		BAN::Variant<ConnectionInfo, ConnectionlessInfo> m_info;

kernel/include/kernel/PCI.h

@@ -66,11 +66,13 @@ namespace Kernel::PCI
 		};
 
 	public:
-		Device() = default;
+		Device(uint8_t bus, uint8_t dev, uint8_t func)
+			: m_bus(bus)
+			, m_dev(dev)
+			, m_func(func)
+		{ }
 
-		void set_location(uint8_t bus, uint8_t dev, uint8_t func);
 		void initialize(paddr_t pcie_paddr);
-		bool is_valid() const { return m_is_valid; }
 
 		uint32_t read_dword(uint8_t) const;
 		uint16_t read_word(uint8_t) const;
@@ -124,10 +126,9 @@ namespace Kernel::PCI
 		BAN::ErrorOr<uint8_t> find_intx_interrupt();
 
 	private:
-		bool m_is_valid	{ false };
-		uint8_t m_bus	{ 0 };
-		uint8_t m_dev	{ 0 };
-		uint8_t m_func	{ 0 };
+		const uint8_t m_bus	{ 0 };
+		const uint8_t m_dev	{ 0 };
+		const uint8_t m_func	{ 0 };
 
 		vaddr_t m_mmio_config { 0 };
 
@@ -161,11 +162,8 @@ namespace Kernel::PCI
 		template<typename F>
 		void for_each_device(F callback)
 		{
-			for (auto& bus : m_buses)
-				for (auto& dev : bus)
-					for (auto& func : dev)
-						if (func.is_valid())
-							callback(func);
+			for (auto& dev : m_devices)
+				callback(dev);
 		};
 
 		uint32_t read_config_dword(uint8_t bus, uint8_t dev, uint8_t func, uint8_t offset);
@@ -179,19 +177,22 @@ namespace Kernel::PCI
 		BAN::Optional<uint8_t> reserve_msi();
 
 	private:
-		PCIManager() : m_bus_pcie_paddr(0) {}
-		void check_function(uint8_t bus, uint8_t dev, uint8_t func);
-		void check_device(uint8_t bus, uint8_t dev);
-		void check_bus(uint8_t bus);
-		void check_all_buses();
+		struct PCIeInfo
+		{
+			paddr_t bus_paddr[256];
+		};
+
+		PCIManager() = default;
+		void check_function(const PCIeInfo&, uint8_t bus, uint8_t dev, uint8_t func);
+		void check_device(const PCIeInfo&, uint8_t bus, uint8_t dev);
+		void check_bus(const PCIeInfo&, uint8_t bus);
+		void check_all_buses(const PCIeInfo&);
 		void initialize_impl();
 
 	private:
 		static constexpr uint8_t m_msi_count = IRQ_MSI_END - IRQ_MSI_BASE;
-		using PCIBus = BAN::Array<BAN::Array<Device, 8>, 32>;
-		BAN::Array<PCIBus, 256>  m_buses;
-		BAN::Array<paddr_t, 256> m_bus_pcie_paddr;
-		bool                     m_is_pcie { false };
+
+		BAN::Vector<Device> m_devices;
 
 		SpinLock                             m_reserved_msi_lock;
 		BAN::Array<uint8_t, m_msi_count / 8> m_reserved_msi_bitmap;

kernel/include/kernel/Terminal/PseudoTerminal.h

@@ -67,7 +67,7 @@ namespace Kernel
 		bool putchar_impl(uint8_t ch) override;
 
 		bool can_write_impl() const override;
-		bool has_hungup_impl() const override { return !m_master.valid(); }
+		bool has_hungup_impl() const override { return master_has_closed(); }
 
 	private:
 		PseudoTerminalSlave(BAN::String&& name, uint32_t number, mode_t, uid_t, gid_t);

kernel/include/kernel/Terminal/TTY.h

@@ -46,12 +46,14 @@ namespace Kernel
 
 		static void keyboard_task(void*);
 		static void initialize_devices();
+
+		bool should_receive_input() const { return m_tty_ctrl.receive_input; }
+		void on_key_event(LibInput::RawKeyEvent);
 		void on_key_event(LibInput::KeyEvent);
 		void handle_input_byte(uint8_t);
 
-		void get_termios(termios* termios) { *termios = m_termios; }
-		// FIXME: validate termios
-		BAN::ErrorOr<void> set_termios(const termios* termios) { m_termios = *termios; return {}; }
+		void get_termios(termios*);
+		BAN::ErrorOr<void> set_termios(const termios*);
 
 		virtual bool is_tty() const override { return true; }
 
@@ -85,9 +87,6 @@ namespace Kernel
 		bool putchar(uint8_t ch);
 		void do_backspace();
 
-	protected:
-		termios m_termios;
-
 	private:
 		const dev_t m_rdev;
 
@@ -95,23 +94,27 @@ namespace Kernel
 
 		struct tty_ctrl_t
 		{
-			bool draw_graphics { true };
-			bool receive_input { true };
-			ThreadBlocker thread_blocker;
+			BAN::Atomic<bool> draw_graphics { true };
+			BAN::Atomic<bool> receive_input { true };
 		};
 		tty_ctrl_t m_tty_ctrl;
 
 		struct Buffer
 		{
 			BAN::UniqPtr<ByteRingBuffer> buffer;
-			bool flush { false };
+			BAN::Atomic<bool> flush { false };
 			ThreadBlocker thread_blocker;
 		};
 		Buffer m_output;
 
 		winsize m_winsize {};
 
+		SpinLock m_termios_lock;
+		termios m_termios;
+
 	protected:
+		Mutex m_mutex;
+
 		RecursiveSpinLock m_write_lock;
 		ThreadBlocker m_write_blocker;
 	};

kernel/include/kernel/USB/HID/Joystick.h

@@ -62,8 +62,8 @@ namespace Kernel
 		Mutex m_command_mutex;
 
 		BAN::Atomic<bool> m_has_initialized_leds { false };
-		uint8_t m_led_state       { 0b0001 };
-		uint8_t m_rumble_strength { 0x00 };
+		BAN::Atomic<uint8_t> m_led_state         { 0b0001 };
+		BAN::Atomic<uint8_t> m_rumble_strength   { 0x00 };
 
 		friend class BAN::RefPtr<USBJoystick>;
 	};

kernel/include/kernel/USB/HID/Keyboard.h

@@ -11,6 +11,8 @@ namespace Kernel
 		BAN_NON_MOVABLE(USBKeyboard);
 
 	public:
+		BAN::ErrorOr<void> initialize() override;
+
 		void start_report() override;
 		void stop_report() override;
 
@@ -38,6 +40,8 @@ namespace Kernel
 		uint16_t m_toggle_mask { 0 };
 
 		uint16_t m_led_mask { 0 };
+		BAN::UniqPtr<DMARegion> m_led_region;
+
 		BAN::Vector<USBHID::Report> m_outputs;
 
 		BAN::Optional<uint8_t> m_repeat_scancode;

kernel/kernel/ACPI/BatterySystem.cpp

@@ -57,7 +57,7 @@ namespace Kernel::ACPI
 				m_last_value = target_conv.value().as.integer.value;
 			}
 
-			auto target_str = TRY(BAN::String::formatted("{}", m_last_value));
+			auto target_str = TRY(BAN::String::formatted("{}", m_last_value.load()));
 
 			if (static_cast<size_t>(offset) >= target_str.size())
 				return 0;
@@ -67,8 +67,8 @@ namespace Kernel::ACPI
 			return ncopy;
 		}
 
-		BAN::ErrorOr<size_t> write_impl(off_t, BAN::ConstByteSpan) override		{ return BAN::Error::from_errno(EINVAL); }
-		BAN::ErrorOr<void> truncate_impl(size_t) override						{ return BAN::Error::from_errno(EINVAL); }
+		BAN::ErrorOr<size_t> write_impl(off_t, BAN::ConstByteSpan) override { return BAN::Error::from_errno(EINVAL); }
+		BAN::ErrorOr<void> truncate_impl(size_t) override { return BAN::Error::from_errno(EINVAL); }
 
 		bool can_read_impl() const override { return true; }
 		bool can_write_impl() const override { return false; }
@@ -90,8 +90,8 @@ namespace Kernel::ACPI
 		AML::NameString m_method_name;
 		size_t m_result_index;
 
-		uint64_t m_last_read_ms = 0;
-		uint64_t m_last_value = 0;
+		BAN::Atomic<uint64_t> m_last_read_ms = 0;
+		BAN::Atomic<uint64_t> m_last_value = 0;
 	};
 
 	BAN::ErrorOr<void> BatterySystem::initialize(AML::Namespace& acpi_namespace)

kernel/kernel/Epoll.cpp

@@ -153,8 +153,6 @@ namespace Kernel
 						REMOVE_IT();
 
 					{
-						LockGuard inode_locker(inode->m_mutex);
-
 #define CHECK_EVENT_BIT(mask, func) \
 						if ((events & mask) && !inode->func()) \
 							events &= ~mask;

kernel/kernel/FS/DevFS/FileSystem.cpp

@@ -102,7 +102,7 @@ namespace Kernel
 				uint64_t next_sync_ms { sync_interval_ms };
 				while (true)
 				{
-					LockGuard _(devfs->m_device_lock);
+					devfs->m_device_lock.lock();
 					while (!devfs->m_should_sync)
 					{
 						const uint64_t current_ms = SystemTimer::get().ms_since_boot();
@@ -110,12 +110,17 @@ namespace Kernel
 							break;
 						devfs->m_sync_thread_blocker.block_with_timeout_ms(next_sync_ms - current_ms, &devfs->m_device_lock);
 					}
-
+					BAN::Vector<BAN::RefPtr<StorageDevice>> storage_devices;
 					for (auto& device : devfs->m_devices)
 						if (device->is_storage_device())
-							if (auto ret = static_cast<StorageDevice*>(device.ptr())->sync_disk_cache(); ret.is_error())
-								dwarnln("disk sync: {}", ret.error());
+							MUST(storage_devices.push_back(static_cast<StorageDevice*>(device.ptr())));
+					devfs->m_device_lock.unlock();
 
+					for (auto& device : storage_devices)
+						if (auto ret = device->sync_disk_cache(); ret.is_error())
+							dwarnln("disk sync: {}", ret.error());
+
+					LockGuard _(devfs->m_device_lock);
 					next_sync_ms = SystemTimer::get().ms_since_boot() + sync_interval_ms;
 					devfs->m_should_sync = false;
 					devfs->m_sync_done.unblock();

kernel/kernel/FS/EventFD.cpp

@@ -1,4 +1,5 @@
 #include <kernel/FS/EventFD.h>
+#include <kernel/Lock/LockGuard.h>
 
 #include <sys/epoll.h>
 
@@ -18,16 +19,20 @@ namespace Kernel
 		if (buffer.size() < sizeof(uint64_t))
 			return BAN::Error::from_errno(EINVAL);
 
+		LockGuard _(m_mutex);
+
 		while (m_value == 0)
 			TRY(Thread::current().block_or_eintr_indefinite(m_thread_blocker, &m_mutex));
 
-		const uint64_t read_value = m_is_semaphore ? 1 : m_value;
+		const uint64_t read_value = m_is_semaphore ? 1 : m_value.load();
 		m_value -= read_value;
 
 		buffer.as<uint64_t>() = read_value;
 
 		epoll_notify(EPOLLOUT);
 
+		m_thread_blocker.unblock();
+
 		return sizeof(uint64_t);
 	}
 
@@ -40,6 +45,8 @@ namespace Kernel
 		if (write_value == UINT64_MAX)
 			return BAN::Error::from_errno(EINVAL);
 
+		LockGuard _(m_mutex);
+
 		while (m_value + write_value < m_value)
 			TRY(Thread::current().block_or_eintr_indefinite(m_thread_blocker, &m_mutex));
 
@@ -48,6 +55,8 @@ namespace Kernel
 		if (m_value > 0)
 			epoll_notify(EPOLLIN);
 
+		m_thread_blocker.unblock();
+
 		return sizeof(uint64_t);
 	}
 

kernel/kernel/FS/Ext2/FileSystem.cpp

@@ -307,31 +307,27 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Ext2FS::read_block(uint32_t block, BlockBufferWrapper& buffer)
 	{
-		LockGuard _(m_mutex);
-
 		const uint32_t sector_size = m_block_device->blksize();
 		const uint32_t block_size = this->block_size();
 		const uint32_t sectors_per_block = block_size / sector_size;
 
 		ASSERT(block >= superblock().first_data_block + 1);
 		ASSERT(buffer.size() >= block_size);
-		TRY(m_block_device->read_blocks(block * sectors_per_block, sectors_per_block, buffer.span()));
 
+		TRY(m_block_device->read_blocks(block * sectors_per_block, sectors_per_block, buffer.span()));
 		return {};
 	}
 
 	BAN::ErrorOr<void> Ext2FS::write_block(uint32_t block, const BlockBufferWrapper& buffer)
 	{
-		LockGuard _(m_mutex);
-
 		const uint32_t sector_size = m_block_device->blksize();
 		const uint32_t block_size = this->block_size();
 		const uint32_t sectors_per_block = block_size / sector_size;
 
 		ASSERT(block >= superblock().first_data_block + 1);
 		ASSERT(buffer.size() >= block_size);
-		TRY(m_block_device->write_blocks(block * sectors_per_block, sectors_per_block, buffer.span()));
 
+		TRY(m_block_device->write_blocks(block * sectors_per_block, sectors_per_block, buffer.span()));
 		return {};
 	}
 
@@ -339,8 +335,6 @@ namespace Kernel
 	{
 		auto superblock_buffer = TRY(get_block_buffer());
 
-		LockGuard _(m_mutex);
-
 		const uint32_t sector_size = m_block_device->blksize();
 		ASSERT(1024 % sector_size == 0);
 
@@ -364,8 +358,6 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Ext2FS::sync_block(uint32_t block)
 	{
-		LockGuard _(m_mutex);
-
 		const uint32_t sector_size = m_block_device->blksize();
 		const uint32_t block_size = this->block_size();
 		const uint32_t sectors_per_block = block_size / sector_size;

kernel/kernel/FS/Ext2/Inode.cpp

@@ -47,7 +47,7 @@ namespace Kernel
 	{
 		if (m_inode.links_count > 0)
 			return;
-		if (auto ret = cleanup_from_fs(); ret.is_error())
+		if (auto ret = cleanup_from_fs_no_lock(); ret.is_error())
 			dwarnln("Could not cleanup inode from FS: {}", ret.error());
 	}
 
@@ -56,7 +56,7 @@ namespace Kernel
 		return &m_fs;
 	}
 
-	BAN::ErrorOr<BAN::Optional<uint32_t>> Ext2Inode::block_from_indirect_block(uint32_t& block, uint32_t index, uint32_t depth, bool allocate)
+	BAN::ErrorOr<BAN::Optional<uint32_t>> Ext2Inode::block_from_indirect_block_no_lock(uint32_t& block, uint32_t index, uint32_t depth, bool allocate)
 	{
 		const uint32_t inode_blocks_per_fs_block = blksize() / 512;
 		const uint32_t indices_per_fs_block = blksize() / sizeof(uint32_t);
@@ -102,7 +102,7 @@ namespace Kernel
 		uint32_t& new_block = block_buffer.span().as_span<uint32_t>()[(index / divisor) % indices_per_fs_block];
 		const auto old_block = new_block;
 
-		const auto result = TRY(block_from_indirect_block(new_block, index, depth - 1, allocate));
+		const auto result = TRY(block_from_indirect_block_no_lock(new_block, index, depth - 1, allocate));
 
 		if (needs_write || old_block != new_block)
 			TRY(m_fs.write_block(block, block_buffer));
@@ -110,7 +110,7 @@ namespace Kernel
 		return result;
 	}
 
-	BAN::ErrorOr<BAN::Optional<uint32_t>> Ext2Inode::fs_block_of_data_block_index(uint32_t data_block_index, bool allocate)
+	BAN::ErrorOr<BAN::Optional<uint32_t>> Ext2Inode::fs_block_of_data_block_index_no_lock(uint32_t data_block_index, bool allocate)
 	{
 		const uint32_t inode_blocks_per_fs_block = blksize() / 512;
 		const uint32_t indices_per_block = blksize() / sizeof(uint32_t);
@@ -136,15 +136,15 @@ namespace Kernel
 		data_block_index -= 12;
 
 		if (data_block_index < indices_per_block)
-			return block_from_indirect_block(m_inode.block[12], data_block_index, 1, allocate);
+			return block_from_indirect_block_no_lock(m_inode.block[12], data_block_index, 1, allocate);
 		data_block_index -= indices_per_block;
 
 		if (data_block_index < indices_per_block * indices_per_block)
-			return block_from_indirect_block(m_inode.block[13], data_block_index, 2, allocate);
+			return block_from_indirect_block_no_lock(m_inode.block[13], data_block_index, 2, allocate);
 		data_block_index -= indices_per_block * indices_per_block;
 
 		if (data_block_index < indices_per_block * indices_per_block * indices_per_block)
-			return block_from_indirect_block(m_inode.block[14], data_block_index, 3, allocate);
+			return block_from_indirect_block_no_lock(m_inode.block[14], data_block_index, 3, allocate);
 
 		ASSERT_NOT_REACHED();
 	}
@@ -152,6 +152,9 @@ namespace Kernel
 	BAN::ErrorOr<BAN::String> Ext2Inode::link_target_impl()
 	{
 		ASSERT(mode().iflnk());
+
+		RWLockRDGuard _(m_lock);
+
 		if (m_inode.size < sizeof(m_inode.block))
 		{
 			BAN::String result;
@@ -168,14 +171,16 @@ namespace Kernel
 	BAN::ErrorOr<void> Ext2Inode::set_link_target_impl(BAN::StringView target)
 	{
 		ASSERT(mode().iflnk());
+
+		RWLockWRGuard _(m_lock);
 		if (target.size() < sizeof(m_inode.block))
 		{
 			if (m_inode.size >= sizeof(m_inode.block))
-				TRY(cleanup_data_blocks());
+				TRY(cleanup_data_blocks_no_lock());
 			memset(m_inode.block, 0, sizeof(m_inode.block));
 			memcpy(m_inode.block, target.data(), target.size());
 			m_inode.size = target.size();
-			TRY(sync());
+			TRY(sync_no_lock());
 			return {};
 		}
 
@@ -194,10 +199,12 @@ namespace Kernel
 		if (static_cast<BAN::make_unsigned_t<decltype(offset)>>(offset) >= UINT32_MAX || buffer.size() >= UINT32_MAX || buffer.size() >= (size_t)(UINT32_MAX - offset))
 			return BAN::Error::from_errno(EOVERFLOW);
 
+		RWLockRDGuard _0(m_lock);
+
 		if (static_cast<BAN::make_unsigned_t<decltype(offset)>>(offset) >= m_inode.size)
 			return 0;
 
-		ScopedSync _(*this);
+		ScopedSync _1(*this);
 
 		uint32_t count = buffer.size();
 		if (offset + buffer.size() > m_inode.size)
@@ -214,7 +221,7 @@ namespace Kernel
 
 		for (uint32_t data_block_index = first_block; data_block_index < last_block; data_block_index++)
 		{
-			auto block_index = TRY(fs_block_of_data_block_index(data_block_index, false));
+			auto block_index = TRY(fs_block_of_data_block_index_no_lock(data_block_index, false));
 			if (block_index.has_value())
 				TRY(m_fs.read_block(block_index.value(), block_buffer));
 			else
@@ -240,6 +247,8 @@ namespace Kernel
 		if (static_cast<BAN::make_unsigned_t<decltype(offset)>>(offset) >= UINT32_MAX || buffer.size() >= UINT32_MAX || buffer.size() >= (size_t)(UINT32_MAX - offset))
 			return BAN::Error::from_errno(EOVERFLOW);
 
+		RWLockWRGuard _0(m_lock);
+
 		if (m_inode.size < offset + buffer.size())
 			TRY(truncate_impl(offset + buffer.size()));
 
@@ -255,7 +264,7 @@ namespace Kernel
 		// Write partial block
 		if (offset % block_size)
 		{
-			const auto block_index = TRY(fs_block_of_data_block_index(offset / block_size, true));
+			const auto block_index = TRY(fs_block_of_data_block_index_no_lock(offset / block_size, true));
 			ASSERT(block_index.has_value());
 
 			TRY(m_fs.read_block(block_index.value(), block_buffer));
@@ -273,7 +282,7 @@ namespace Kernel
 
 		while (to_write >= block_size)
 		{
-			const auto block_index = TRY(fs_block_of_data_block_index(offset / block_size, true));
+			const auto block_index = TRY(fs_block_of_data_block_index_no_lock(offset / block_size, true));
 			ASSERT(block_index.has_value());
 
 			memcpy(block_buffer.data(), buffer.data() + written, block_buffer.size());
@@ -286,7 +295,7 @@ namespace Kernel
 
 		if (to_write > 0)
 		{
-			const auto block_index = TRY(fs_block_of_data_block_index(offset / block_size, true));
+			const auto block_index = TRY(fs_block_of_data_block_index_no_lock(offset / block_size, true));
 			ASSERT(block_index.has_value());
 
 			TRY(m_fs.read_block(block_index.value(), block_buffer));
@@ -300,6 +309,8 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Ext2Inode::truncate_impl(size_t new_size)
 	{
+		RWLockWRGuard _(m_lock);
+
 		if (m_inode.size == new_size)
 			return {};
 
@@ -308,7 +319,7 @@ namespace Kernel
 		const auto old_size = m_inode.size;
 
 		m_inode.size = new_size;
-		if (auto ret = sync(); ret.is_error())
+		if (auto ret = sync_no_lock(); ret.is_error())
 		{
 			m_inode.size = old_size;
 			return ret.release_error();
@@ -320,13 +331,16 @@ namespace Kernel
 	BAN::ErrorOr<void> Ext2Inode::chmod_impl(mode_t mode)
 	{
 		ASSERT((mode & Inode::Mode::TYPE_MASK) == 0);
+
+		RWLockWRGuard _(m_lock);
+
 		if (m_inode.mode == mode)
 			return {};
 
 		const auto old_mode = m_inode.mode;
 
 		m_inode.mode = (m_inode.mode & Inode::Mode::TYPE_MASK) | mode;
-		if (auto ret = sync(); ret.is_error())
+		if (auto ret = sync_no_lock(); ret.is_error())
 		{
 			m_inode.mode = old_mode;
 			return ret.release_error();
@@ -337,6 +351,8 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Ext2Inode::chown_impl(uid_t uid, gid_t gid)
 	{
+		RWLockWRGuard _(m_lock);
+
 		if (m_inode.uid == uid && m_inode.gid == gid)
 			return {};
 
@@ -345,7 +361,7 @@ namespace Kernel
 
 		m_inode.uid = uid;
 		m_inode.gid = gid;
-		if (auto ret = sync(); ret.is_error())
+		if (auto ret = sync_no_lock(); ret.is_error())
 		{
 			m_inode.uid = old_uid;
 			m_inode.gid = old_gid;
@@ -357,6 +373,8 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Ext2Inode::utimens_impl(const timespec times[2])
 	{
+		RWLockWRGuard _(m_lock);
+
 		const uint32_t old_times[2] {
 			m_inode.atime,
 			m_inode.mtime,
@@ -367,7 +385,7 @@ namespace Kernel
 		if (times[1].tv_nsec != UTIME_OMIT)
 			m_inode.mtime = times[1].tv_sec;
 
-		if (auto ret = sync(); ret.is_error())
+		if (auto ret = sync_no_lock(); ret.is_error())
 		{
 			m_inode.atime = old_times[0];
 			m_inode.mtime = old_times[1];
@@ -379,13 +397,14 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Ext2Inode::fsync_impl()
 	{
+		RWLockRDGuard _(m_lock);
 		for (size_t i = 0; i < max_used_data_block_count(); i++)
-			if (const auto fs_block = TRY(fs_block_of_data_block_index(i, false)); fs_block.has_value())
+			if (const auto fs_block = TRY(fs_block_of_data_block_index_no_lock(i, false)); fs_block.has_value())
 				TRY(m_fs.sync_block(fs_block.value()));
 		return {};
 	}
 
-	BAN::ErrorOr<void> Ext2Inode::cleanup_indirect_block(uint32_t block, uint32_t depth)
+	BAN::ErrorOr<void> Ext2Inode::cleanup_indirect_block_no_lock(uint32_t block, uint32_t depth)
 	{
 		ASSERT(block);
 
@@ -404,14 +423,14 @@ namespace Kernel
 			const uint32_t next_block = block_buffer.span().as_span<uint32_t>()[i];
 			if (next_block == 0)
 				continue;
-			TRY(cleanup_indirect_block(next_block, depth - 1));
+			TRY(cleanup_indirect_block_no_lock(next_block, depth - 1));
 		}
 
 		TRY(m_fs.release_block(block));
 		return {};
 	}
 
-	BAN::ErrorOr<void> Ext2Inode::cleanup_data_blocks()
+	BAN::ErrorOr<void> Ext2Inode::cleanup_data_blocks_no_lock()
 	{
 		if (mode().iflnk() && (size_t)size() < sizeof(m_inode.block))
 			goto done;
@@ -423,25 +442,25 @@ namespace Kernel
 
 		// cleanup indirect blocks
 		if (m_inode.block[12])
-			TRY(cleanup_indirect_block(m_inode.block[12], 1));
+			TRY(cleanup_indirect_block_no_lock(m_inode.block[12], 1));
 		if (m_inode.block[13])
-			TRY(cleanup_indirect_block(m_inode.block[13], 2));
+			TRY(cleanup_indirect_block_no_lock(m_inode.block[13], 2));
 		if (m_inode.block[14])
-			TRY(cleanup_indirect_block(m_inode.block[14], 3));
+			TRY(cleanup_indirect_block_no_lock(m_inode.block[14], 3));
 
 done:
 		// mark blocks as deleted
 		memset(m_inode.block, 0x00, sizeof(m_inode.block));
 
-		TRY(sync());
+		TRY(sync_no_lock());
 
 		return {};
 	}
 
-	BAN::ErrorOr<void> Ext2Inode::cleanup_from_fs()
+	BAN::ErrorOr<void> Ext2Inode::cleanup_from_fs_no_lock()
 	{
 		ASSERT(m_inode.links_count == 0);
-		TRY(cleanup_data_blocks());
+		TRY(cleanup_data_blocks_no_lock());
 		TRY(m_fs.delete_inode(ino()));
 		return {};
 	}
@@ -451,10 +470,12 @@ done:
 		ASSERT(mode().ifdir());
 		ASSERT(offset >= 0);
 
+		RWLockRDGuard _(m_lock);
+
 		if (static_cast<BAN::make_unsigned_t<decltype(offset)>>(offset) >= max_used_data_block_count())
 			return 0;
 
-		const auto block_index = TRY(fs_block_of_data_block_index(offset, false));
+		const auto block_index = TRY(fs_block_of_data_block_index_no_lock(offset, false));
 		if (!block_index.has_value())
 			return BAN::Error::from_errno(ENODATA);
 
@@ -524,9 +545,8 @@ done:
 	{
 		ASSERT(mode_has_valid_type(mode));
 
-		timespec current_time = SystemTimer::get().real_time();
-		return Ext2::Inode
-		{
+		const timespec current_time = SystemTimer::get().real_time();
+		return Ext2::Inode {
 			.mode			= (uint16_t)mode,
 			.uid			= (uint16_t)uid,
 			.size			= 0,
@@ -552,9 +572,6 @@ done:
 	{
 		ASSERT(this->mode().ifdir());
 
-		if (!find_inode_impl(name).is_error())
-			return BAN::Error::from_errno(EEXIST);
-
 		switch (mode & Inode::Mode::TYPE_MASK)
 		{
 			case Inode::Mode::IFLNK:
@@ -565,6 +582,11 @@ done:
 			return BAN::Error::from_errno(ENOTSUP);
 		}
 
+		RWLockWRGuard _(m_lock);
+
+		if (!find_inode_no_lock(name).is_error())
+			return BAN::Error::from_errno(EEXIST);
+
 		const uint32_t new_ino = TRY(m_fs.create_inode(initialize_new_inode_info(mode, uid, gid)));
 
 		auto inode_or_error = Ext2Inode::create(m_fs, new_ino);
@@ -576,7 +598,7 @@ done:
 
 		auto inode = inode_or_error.release_value();
 
-		TRY(link_inode_to_directory(*inode, name));
+		TRY(link_inode_to_directory_no_lock(*inode, name));
 
 		return {};
 	}
@@ -586,7 +608,9 @@ done:
 		ASSERT(this->mode().ifdir());
 		ASSERT(Mode(mode).ifdir());
 
-		if (!find_inode_impl(name).is_error())
+		RWLockWRGuard _(m_lock);
+
+		if (!find_inode_no_lock(name).is_error())
 			return BAN::Error::from_errno(EEXIST);
 
 		const uint32_t new_ino = TRY(m_fs.create_inode(initialize_new_inode_info(mode, uid, gid)));
@@ -601,14 +625,14 @@ done:
 		auto inode = inode_or_error.release_value();
 
 		// link . and ..
-		if (auto ret = inode->link_inode_to_directory(*inode, "."_sv); ret.is_error())
-			return ({ TRY(inode->cleanup_from_fs()); ret.release_error(); });
-		if (auto ret = inode->link_inode_to_directory(*this, ".."_sv); ret.is_error())
-			return ({ TRY(inode->cleanup_from_fs()); ret.release_error(); });
+		if (auto ret = inode->link_inode_to_directory_no_lock(*inode, "."_sv); ret.is_error())
+			return ({ TRY(inode->cleanup_from_fs_no_lock()); ret.release_error(); });
+		if (auto ret = inode->link_inode_to_directory_no_lock(*this, ".."_sv); ret.is_error())
+			return ({ TRY(inode->cleanup_from_fs_no_lock()); ret.release_error(); });
 
 		// link to parent
-		if (auto ret = link_inode_to_directory(*inode, name); ret.is_error())
-			return ({ TRY(inode->cleanup_from_fs()); ret.release_error(); });
+		if (auto ret = link_inode_to_directory_no_lock(*inode, name); ret.is_error())
+			return ({ TRY(inode->cleanup_from_fs_no_lock()); ret.release_error(); });
 
 		return {};
 	}
@@ -619,11 +643,13 @@ done:
 		ASSERT(!inode->mode().ifdir());
 		ASSERT(&m_fs == inode->filesystem());
 
-		if (!find_inode_impl(name).is_error())
+		RWLockWRGuard _(m_lock);
+
+		if (!find_inode_no_lock(name).is_error())
 			return BAN::Error::from_errno(EEXIST);
 
 		auto ext2_inode = static_cast<Ext2Inode*>(inode.ptr());
-		TRY(link_inode_to_directory(*ext2_inode, name));
+		TRY(link_inode_to_directory_no_lock(*ext2_inode, name));
 
 		return {};
 	}
@@ -636,30 +662,31 @@ done:
 
 		auto* ext2_parent = static_cast<Ext2Inode*>(old_parent.ptr());
 
-		// FIXME: possible deadlock :)
-		LockGuard _(ext2_parent->m_mutex);
+		// FIXME: is this a possible deadlock?
+		RWLockWRGuard _0(ext2_parent->m_lock);
+		RWLockWRGuard _1(m_lock);
 
-		auto old_inode = TRY(ext2_parent->find_inode_impl(old_name));
+		auto old_inode = TRY(ext2_parent->find_inode_no_lock(old_name));
 		auto* ext2_inode = static_cast<Ext2Inode*>(old_inode.ptr());
 
-		if (auto replace_or_error = find_inode_impl(new_name); replace_or_error.is_error())
+		if (auto find_result = find_inode_no_lock(new_name); find_result.is_error())
 		{
-			if (replace_or_error.error().get_error_code() != ENOENT)
-				return replace_or_error.release_error();
+			if (find_result.error().get_error_code() != ENOENT)
+				return find_result.release_error();
 		}
 		else
 		{
-			TRY(unlink_impl(new_name));
+			TRY(remove_inode_from_directory_no_lock(new_name, true));
 		}
 
-		TRY(link_inode_to_directory(*ext2_inode, new_name));
+		TRY(link_inode_to_directory_no_lock(*ext2_inode, new_name));
 
-		TRY(ext2_parent->remove_inode_from_directory(old_name, false));
+		TRY(ext2_parent->remove_inode_from_directory_no_lock(old_name, false));
 
 		return {};
 	}
 
-	BAN::ErrorOr<void> Ext2Inode::link_inode_to_directory(Ext2Inode& inode, BAN::StringView name)
+	BAN::ErrorOr<void> Ext2Inode::link_inode_to_directory_no_lock(Ext2Inode& inode, BAN::StringView name)
 	{
 		if (!this->mode().ifdir())
 			return BAN::Error::from_errno(ENOTDIR);
@@ -673,7 +700,7 @@ done:
 			return BAN::Error::from_errno(ENOTSUP);
 		}
 
-		auto error_or = find_inode_impl(name);
+		auto error_or = find_inode_no_lock(name);
 		if (!error_or.is_error())
 			return BAN::Error::from_errno(EEXIST);
 		if (error_or.error().get_error_code() != ENOENT)
@@ -705,7 +732,7 @@ done:
 				memcpy(new_entry.name, name.data(), name.size());
 
 				inode.m_inode.links_count++;
-				TRY(inode.sync());
+				TRY(inode.sync_no_lock());
 
 				return {};
 			};
@@ -723,7 +750,7 @@ done:
 			goto needs_new_block;
 
 		// Try to insert inode to last data block
-		block_index = TRY(fs_block_of_data_block_index(data_block_count - 1, true)).value();
+		block_index = TRY(fs_block_of_data_block_index_no_lock(data_block_count - 1, true)).value();
 		TRY(m_fs.read_block(block_index, block_buffer));
 
 		while (entry_offset < block_size)
@@ -754,7 +781,7 @@ done:
 		}
 
 needs_new_block:
-		block_index = TRY(fs_block_of_data_block_index(data_block_count, true)).value();
+		block_index = TRY(fs_block_of_data_block_index_no_lock(data_block_count, true)).value();
 		m_inode.size += blksize();
 
 		memset(block_buffer.data(), 0x00, block_buffer.size());
@@ -764,7 +791,7 @@ needs_new_block:
 		return {};
 	}
 
-	BAN::ErrorOr<bool> Ext2Inode::is_directory_empty()
+	BAN::ErrorOr<bool> Ext2Inode::is_directory_empty_no_lock()
 	{
 		ASSERT(mode().ifdir());
 
@@ -773,7 +800,7 @@ needs_new_block:
 		// Confirm that this doesn't contain anything else than '.' or '..'
 		for (uint32_t i = 0; i < max_used_data_block_count(); i++)
 		{
-			const auto block_index = TRY(fs_block_of_data_block_index(i, false));
+			const auto block_index = TRY(fs_block_of_data_block_index_no_lock(i, false));
 			if (!block_index.has_value())
 				continue;
 			TRY(m_fs.read_block(block_index.value(), block_buffer));
@@ -797,20 +824,21 @@ needs_new_block:
 		return true;
 	}
 
-	BAN::ErrorOr<void> Ext2Inode::cleanup_default_links()
+	BAN::ErrorOr<void> Ext2Inode::cleanup_default_links_no_lock()
 	{
 		ASSERT(mode().ifdir());
+
+		auto block_buffer = TRY(m_fs.get_block_buffer());
+
 		if (m_inode.flags & Ext2::Enum::INDEX_FL)
 		{
 			dwarnln("deletion of indexed directory is not supported");
 			return BAN::Error::from_errno(ENOTSUP);
 		}
 
-		auto block_buffer = TRY(m_fs.get_block_buffer());
-
 		for (uint32_t i = 0; i < max_used_data_block_count(); i++)
 		{
-			const auto block_index = TRY(fs_block_of_data_block_index(i, false));
+			const auto block_index = TRY(fs_block_of_data_block_index_no_lock(i, false));
 			if (!block_index.has_value())
 				continue;
 			TRY(m_fs.read_block(block_index.value(), block_buffer));
@@ -829,13 +857,13 @@ needs_new_block:
 					if (entry_name == "."_sv)
 					{
 						m_inode.links_count--;
-						TRY(sync());
+						TRY(sync_no_lock());
 					}
 					else if (entry_name == ".."_sv)
 					{
 						auto parent = TRY(Ext2Inode::create(m_fs, entry.inode));
 						parent->m_inode.links_count--;
-						TRY(parent->sync());
+						TRY(parent->sync_no_lock());
 					}
 					else
 						ASSERT_NOT_REACHED();
@@ -854,20 +882,21 @@ needs_new_block:
 		return {};
 	}
 
-	BAN::ErrorOr<void> Ext2Inode::remove_inode_from_directory(BAN::StringView name, bool cleanup_directory)
+	BAN::ErrorOr<void> Ext2Inode::remove_inode_from_directory_no_lock(BAN::StringView name, bool cleanup_directory)
 	{
 		ASSERT(mode().ifdir());
+
+		auto block_buffer = TRY(m_fs.get_block_buffer());
+
 		if (m_inode.flags & Ext2::Enum::INDEX_FL)
 		{
 			dwarnln("deletion from indexed directory is not supported");
 			return BAN::Error::from_errno(ENOTSUP);
 		}
 
-		auto block_buffer = TRY(m_fs.get_block_buffer());
-
 		for (uint32_t i = 0; i < max_used_data_block_count(); i++)
 		{
-			const auto block_index = TRY(fs_block_of_data_block_index(i, false));
+			const auto block_index = TRY(fs_block_of_data_block_index_no_lock(i, false));
 			if (!block_index.has_value())
 				continue;
 			TRY(m_fs.read_block(block_index.value(), block_buffer));
@@ -881,9 +910,9 @@ needs_new_block:
 					auto inode = TRY(Ext2Inode::create(m_fs, entry.inode));
 					if (cleanup_directory && inode->mode().ifdir())
 					{
-						if (!TRY(inode->is_directory_empty()))
+						if (!TRY(inode->is_directory_empty_no_lock()))
 							return BAN::Error::from_errno(ENOTEMPTY);
-						TRY(inode->cleanup_default_links());
+						TRY(inode->cleanup_default_links_no_lock());
 					}
 
 					if (inode->nlink() == 0)
@@ -891,7 +920,7 @@ needs_new_block:
 					else
 						inode->m_inode.links_count--;
 
-					TRY(sync());
+					TRY(sync_no_lock());
 
 					// NOTE: If this was the last link to inode we must
 					//       remove it from inode cache to trigger cleanup
@@ -915,11 +944,12 @@ needs_new_block:
 
 	BAN::ErrorOr<void> Ext2Inode::unlink_impl(BAN::StringView name)
 	{
-		TRY(remove_inode_from_directory(name, true));
+		RWLockWRGuard _(m_lock);
+		TRY(remove_inode_from_directory_no_lock(name, true));
 		return {};
 	}
 
-	BAN::ErrorOr<void> Ext2Inode::sync()
+	BAN::ErrorOr<void> Ext2Inode::sync_no_lock()
 	{
 		auto inode_location = TRY(m_fs.locate_inode(ino()));
 		auto block_buffer = TRY(m_fs.get_block_buffer());
@@ -935,6 +965,12 @@ needs_new_block:
 	}
 
 	BAN::ErrorOr<BAN::RefPtr<Inode>> Ext2Inode::find_inode_impl(BAN::StringView file_name)
+	{
+		RWLockRDGuard _(m_lock);
+		return find_inode_no_lock(file_name);
+	}
+
+	BAN::ErrorOr<BAN::RefPtr<Inode>> Ext2Inode::find_inode_no_lock(BAN::StringView file_name)
 	{
 		ASSERT(mode().ifdir());
 
@@ -942,7 +978,7 @@ needs_new_block:
 
 		for (uint32_t i = 0; i < max_used_data_block_count(); i++)
 		{
-			const auto block_index = TRY(fs_block_of_data_block_index(i, false));
+			const auto block_index = TRY(fs_block_of_data_block_index_no_lock(i, false));
 			if (!block_index.has_value())
 				continue;
 			TRY(m_fs.read_block(block_index.value(), block_buffer));

kernel/kernel/FS/Inode.cpp

@@ -62,7 +62,6 @@ namespace Kernel
 
 	BAN::ErrorOr<BAN::RefPtr<Inode>> Inode::find_inode(BAN::StringView name)
 	{
-		LockGuard _(m_mutex);
 		if (!mode().ifdir())
 			return BAN::Error::from_errno(ENOTDIR);
 		return find_inode_impl(name);
@@ -70,7 +69,6 @@ namespace Kernel
 
 	BAN::ErrorOr<size_t> Inode::list_next_inodes(off_t offset, struct dirent* list, size_t list_len)
 	{
-		LockGuard _(m_mutex);
 		if (!mode().ifdir())
 			return BAN::Error::from_errno(ENOTDIR);
 		return list_next_inodes_impl(offset, list, list_len);
@@ -78,7 +76,6 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Inode::create_file(BAN::StringView name, mode_t mode, uid_t uid, gid_t gid)
 	{
-		LockGuard _(m_mutex);
 		if (!this->mode().ifdir())
 			return BAN::Error::from_errno(ENOTDIR);
 		if (Mode(mode).ifdir())
@@ -90,7 +87,6 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Inode::create_directory(BAN::StringView name, mode_t mode, uid_t uid, gid_t gid)
 	{
-		LockGuard _(m_mutex);
 		if (!this->mode().ifdir())
 			return BAN::Error::from_errno(ENOTDIR);
 		if (!Mode(mode).ifdir())
@@ -102,7 +98,6 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Inode::link_inode(BAN::StringView name, BAN::RefPtr<Inode> inode)
 	{
-		LockGuard _(m_mutex);
 		if (!this->mode().ifdir())
 			return BAN::Error::from_errno(ENOTDIR);
 		if (inode->mode().ifdir())
@@ -116,7 +111,6 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Inode::rename_inode(BAN::RefPtr<Inode> old_parent, BAN::StringView old_name, BAN::StringView new_name)
 	{
-		LockGuard _(m_mutex);
 		if (!this->mode().ifdir())
 			return BAN::Error::from_errno(ENOTDIR);
 		if (!old_parent->mode().ifdir())
@@ -130,7 +124,6 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Inode::unlink(BAN::StringView name)
 	{
-		LockGuard _(m_mutex);
 		if (!mode().ifdir())
 			return BAN::Error::from_errno(ENOTDIR);
 		if (name == "."_sv || name == ".."_sv)
@@ -142,7 +135,6 @@ namespace Kernel
 
 	BAN::ErrorOr<BAN::String> Inode::link_target()
 	{
-		LockGuard _(m_mutex);
 		if (!mode().iflnk())
 			return BAN::Error::from_errno(EINVAL);
 		return link_target_impl();
@@ -150,7 +142,6 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Inode::set_link_target(BAN::StringView target)
 	{
-		LockGuard _(m_mutex);
 		if (!mode().iflnk())
 			return BAN::Error::from_errno(EINVAL);
 		if (auto* fs = filesystem(); fs && (fs->flag() & ST_RDONLY))
@@ -160,7 +151,6 @@ namespace Kernel
 
 	BAN::ErrorOr<long> Inode::accept(sockaddr* address, socklen_t* address_len, int flags)
 	{
-		LockGuard _(m_mutex);
 		if (!mode().ifsock())
 			return BAN::Error::from_errno(ENOTSOCK);
 		return accept_impl(address, address_len, flags);
@@ -168,7 +158,6 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Inode::bind(const sockaddr* address, socklen_t address_len)
 	{
-		LockGuard _(m_mutex);
 		if (!mode().ifsock())
 			return BAN::Error::from_errno(ENOTSOCK);
 		return bind_impl(address, address_len);
@@ -176,7 +165,6 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Inode::connect(const sockaddr* address, socklen_t address_len)
 	{
-		LockGuard _(m_mutex);
 		if (!mode().ifsock())
 			return BAN::Error::from_errno(ENOTSOCK);
 		return connect_impl(address, address_len);
@@ -184,7 +172,6 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Inode::listen(int backlog)
 	{
-		LockGuard _(m_mutex);
 		if (!mode().ifsock())
 			return BAN::Error::from_errno(ENOTSOCK);
 		return listen_impl(backlog);
@@ -192,7 +179,6 @@ namespace Kernel
 
 	BAN::ErrorOr<size_t> Inode::recvmsg(msghdr& message, int flags)
 	{
-		LockGuard _(m_mutex);
 		if (!mode().ifsock())
 			return BAN::Error::from_errno(ENOTSOCK);
 		return recvmsg_impl(message, flags);
@@ -200,7 +186,6 @@ namespace Kernel
 
 	BAN::ErrorOr<size_t> Inode::sendmsg(const msghdr& message, int flags)
 	{
-		LockGuard _(m_mutex);
 		if (!mode().ifsock())
 			return BAN::Error::from_errno(ENOTSOCK);
 		return sendmsg_impl(message, flags);
@@ -208,7 +193,6 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Inode::getsockname(sockaddr* address, socklen_t* address_len)
 	{
-		LockGuard _(m_mutex);
 		if (!mode().ifsock())
 			return BAN::Error::from_errno(ENOTSOCK);
 		return getsockname_impl(address, address_len);
@@ -216,7 +200,6 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Inode::getpeername(sockaddr* address, socklen_t* address_len)
 	{
-		LockGuard _(m_mutex);
 		if (!mode().ifsock())
 			return BAN::Error::from_errno(ENOTSOCK);
 		return getpeername_impl(address, address_len);
@@ -224,7 +207,6 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Inode::getsockopt(int level, int option, void* value, socklen_t* value_len)
 	{
-		LockGuard _(m_mutex);
 		if (!mode().ifsock())
 			return BAN::Error::from_errno(ENOTSOCK);
 		return getsockopt_impl(level, option, value, value_len);
@@ -232,7 +214,6 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Inode::setsockopt(int level, int option, const void* value, socklen_t value_len)
 	{
-		LockGuard _(m_mutex);
 		if (!mode().ifsock())
 			return BAN::Error::from_errno(ENOTSOCK);
 		return setsockopt_impl(level, option, value, value_len);
@@ -240,7 +221,6 @@ namespace Kernel
 
 	BAN::ErrorOr<size_t> Inode::read(off_t offset, BAN::ByteSpan buffer)
 	{
-		LockGuard _(m_mutex);
 		if (mode().ifdir())
 			return BAN::Error::from_errno(EISDIR);
 		return read_impl(offset, buffer);
@@ -248,7 +228,6 @@ namespace Kernel
 
 	BAN::ErrorOr<size_t> Inode::write(off_t offset, BAN::ConstByteSpan buffer)
 	{
-		LockGuard _(m_mutex);
 		if (mode().ifdir())
 			return BAN::Error::from_errno(EISDIR);
 		if (auto* fs = filesystem(); fs && (fs->flag() & ST_RDONLY))
@@ -258,7 +237,6 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Inode::truncate(size_t size)
 	{
-		LockGuard _(m_mutex);
 		if (mode().ifdir())
 			return BAN::Error::from_errno(EISDIR);
 		if (auto* fs = filesystem(); fs && (fs->flag() & ST_RDONLY))
@@ -269,7 +247,6 @@ namespace Kernel
 	BAN::ErrorOr<void> Inode::chmod(mode_t mode)
 	{
 		ASSERT((mode & Inode::Mode::TYPE_MASK) == 0);
-		LockGuard _(m_mutex);
 		if (auto* fs = filesystem(); fs && (fs->flag() & ST_RDONLY))
 			return BAN::Error::from_errno(EROFS);
 		return chmod_impl(mode);
@@ -277,7 +254,6 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Inode::chown(uid_t uid, gid_t gid)
 	{
-		LockGuard _(m_mutex);
 		if (auto* fs = filesystem(); fs && (fs->flag() & ST_RDONLY))
 			return BAN::Error::from_errno(EROFS);
 		return chown_impl(uid, gid);
@@ -285,7 +261,6 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Inode::utimens(const timespec times[2])
 	{
-		LockGuard _(m_mutex);
 		if (auto* fs = filesystem(); fs && (fs->flag() & ST_RDONLY))
 			return BAN::Error::from_errno(EROFS);
 		return utimens_impl(times);
@@ -293,40 +268,32 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Inode::fsync()
 	{
-		LockGuard _(m_mutex);
-		if (auto shared = m_shared_region.lock())
-			for (size_t i = 0; i < shared->pages.size(); i++)
-				shared->sync(i);
+		// TODO: should we sync shared data?
 		return fsync_impl();
 	}
 
 	bool Inode::can_read() const
 	{
-		LockGuard _(m_mutex);
 		return can_read_impl();
 	}
 
 	bool Inode::can_write() const
 	{
-		LockGuard _(m_mutex);
 		return can_write_impl();
 	}
 
 	bool Inode::has_error() const
 	{
-		LockGuard _(m_mutex);
 		return has_error_impl();
 	}
 
 	bool Inode::has_hungup() const
 	{
-		LockGuard _(m_mutex);
 		return has_hungup_impl();
 	}
 
 	BAN::ErrorOr<long> Inode::ioctl(int request, void* arg)
 	{
-		LockGuard _(m_mutex);
 		return ioctl_impl(request, arg);
 	}
 

kernel/kernel/FS/Pipe.cpp

@@ -5,6 +5,7 @@
 
 #include <fcntl.h>
 #include <sys/epoll.h>
+#include <sys/ioctl.h>
 
 namespace Kernel
 {
@@ -73,6 +74,8 @@ namespace Kernel
 
 	BAN::ErrorOr<size_t> Pipe::read_impl(off_t, BAN::ByteSpan buffer)
 	{
+		LockGuard _(m_mutex);
+
 		while (m_buffer->empty())
 		{
 			if (m_writing_count == 0)
@@ -95,6 +98,8 @@ namespace Kernel
 
 	BAN::ErrorOr<size_t> Pipe::write_impl(off_t, BAN::ConstByteSpan buffer)
 	{
+		LockGuard _(m_mutex);
+
 		while (m_buffer->full())
 		{
 			if (m_reading_count == 0)
@@ -119,4 +124,16 @@ namespace Kernel
 		return to_copy;
 	}
 
+
+	BAN::ErrorOr<long> Pipe::ioctl_impl(int cmd, void* arg)
+	{
+		switch (cmd)
+		{
+			case TIOCGWINSZ:
+			case TIOCSWINSZ:
+				return BAN::Error::from_errno(EINVAL);
+		}
+		return Inode::ioctl_impl(cmd, arg);
+	}
+
 }

kernel/kernel/FS/TmpFS/Inode.cpp

@@ -94,6 +94,7 @@ namespace Kernel
 
 	BAN::ErrorOr<void> TmpInode::chmod_impl(mode_t new_mode)
 	{
+		// FIXME: make this atomic
 		ASSERT(!(new_mode & Inode::Mode::TYPE_MASK));
 		m_inode_info.mode &= Inode::Mode::TYPE_MASK;
 		m_inode_info.mode |= new_mode;
@@ -102,6 +103,7 @@ namespace Kernel
 
 	BAN::ErrorOr<void> TmpInode::chown_impl(uid_t new_uid, gid_t new_gid)
 	{
+		// FIXME: make this atomic
 		m_inode_info.uid = new_uid;
 		m_inode_info.gid = new_gid;
 		return {};
@@ -109,6 +111,7 @@ namespace Kernel
 
 	BAN::ErrorOr<void> TmpInode::utimens_impl(const timespec times[2])
 	{
+		// FIXME: make this atomic
 		if (times[0].tv_nsec != UTIME_OMIT)
 			m_inode_info.atime = times[0];
 		if (times[1].tv_nsec != UTIME_OMIT)
@@ -123,23 +126,24 @@ namespace Kernel
 
 	void TmpInode::free_all_blocks()
 	{
+		LockGuard _(m_lock);
 		if (mode().iflnk() && m_inode_info.size <= sizeof(TmpInodeInfo::block))
 			goto free_all_blocks_done;
 		for (size_t i = 0; i < TmpInodeInfo::direct_block_count; i++)
 			if (m_inode_info.block[i])
 				m_fs.free_block(m_inode_info.block[i]);
 		if (size_t block = m_inode_info.block[TmpInodeInfo::direct_block_count + 0])
-			free_indirect_blocks(block, 1);
+			free_indirect_blocks_no_lock(block, 1);
 		if (size_t block = m_inode_info.block[TmpInodeInfo::direct_block_count + 1])
-			free_indirect_blocks(block, 2);
+			free_indirect_blocks_no_lock(block, 2);
 		if (size_t block = m_inode_info.block[TmpInodeInfo::direct_block_count + 2])
-			free_indirect_blocks(block, 3);
+			free_indirect_blocks_no_lock(block, 3);
 	free_all_blocks_done:
 		for (auto& block : m_inode_info.block)
 			block = 0;
 	}
 
-	void TmpInode::free_indirect_blocks(size_t block, uint32_t depth)
+	void TmpInode::free_indirect_blocks_no_lock(size_t block, uint32_t depth)
 	{
 		ASSERT(block != 0);
 
@@ -160,7 +164,7 @@ namespace Kernel
 			if (next_block == 0)
 				continue;
 
-			free_indirect_blocks(next_block, depth - 1);
+			free_indirect_blocks_no_lock(next_block, depth - 1);
 		}
 
 		m_fs.free_block(block);
@@ -168,6 +172,8 @@ namespace Kernel
 
 	BAN::Optional<size_t> TmpInode::block_index(size_t data_block_index)
 	{
+		LockGuard _(m_lock);
+
 		if (data_block_index < TmpInodeInfo::direct_block_count)
 		{
 			if (m_inode_info.block[data_block_index] == 0)
@@ -179,20 +185,20 @@ namespace Kernel
 		const size_t indices_per_block = blksize() / sizeof(size_t);
 
 		if (data_block_index < indices_per_block)
-			return block_index_from_indirect(m_inode_info.block[TmpInodeInfo::direct_block_count + 0], data_block_index, 1);
+			return block_index_from_indirect_no_lock(m_inode_info.block[TmpInodeInfo::direct_block_count + 0], data_block_index, 1);
 		data_block_index -= indices_per_block;
 
 		if (data_block_index < indices_per_block * indices_per_block)
-			return block_index_from_indirect(m_inode_info.block[TmpInodeInfo::direct_block_count + 1], data_block_index, 2);
+			return block_index_from_indirect_no_lock(m_inode_info.block[TmpInodeInfo::direct_block_count + 1], data_block_index, 2);
 		data_block_index -= indices_per_block * indices_per_block;
 
 		if (data_block_index < indices_per_block * indices_per_block * indices_per_block)
-			return block_index_from_indirect(m_inode_info.block[TmpInodeInfo::direct_block_count + 2], data_block_index, 3);
+			return block_index_from_indirect_no_lock(m_inode_info.block[TmpInodeInfo::direct_block_count + 2], data_block_index, 3);
 
 		ASSERT_NOT_REACHED();
 	}
 
-	BAN::Optional<size_t> TmpInode::block_index_from_indirect(size_t block, size_t index, uint32_t depth)
+	BAN::Optional<size_t> TmpInode::block_index_from_indirect_no_lock(size_t block, size_t index, uint32_t depth)
 	{
 		if (block == 0)
 			return {};
@@ -215,11 +221,13 @@ namespace Kernel
 		if (depth == 1)
 			return next_block;
 
-		return block_index_from_indirect(next_block, index, depth - 1);
+		return block_index_from_indirect_no_lock(next_block, index, depth - 1);
 	}
 
 	BAN::ErrorOr<size_t> TmpInode::block_index_with_allocation(size_t data_block_index)
 	{
+		LockGuard _(m_lock);
+
 		if (data_block_index < TmpInodeInfo::direct_block_count)
 		{
 			if (m_inode_info.block[data_block_index] == 0)
@@ -234,20 +242,20 @@ namespace Kernel
 		const size_t indices_per_block = blksize() / sizeof(size_t);
 
 		if (data_block_index < indices_per_block)
-			return block_index_from_indirect_with_allocation(m_inode_info.block[TmpInodeInfo::direct_block_count + 0], data_block_index, 1);
+			return block_index_from_indirect_with_allocation_no_lock(m_inode_info.block[TmpInodeInfo::direct_block_count + 0], data_block_index, 1);
 		data_block_index -= indices_per_block;
 
 		if (data_block_index < indices_per_block * indices_per_block)
-			return block_index_from_indirect_with_allocation(m_inode_info.block[TmpInodeInfo::direct_block_count + 1], data_block_index, 2);
+			return block_index_from_indirect_with_allocation_no_lock(m_inode_info.block[TmpInodeInfo::direct_block_count + 1], data_block_index, 2);
 		data_block_index -= indices_per_block * indices_per_block;
 
 		if (data_block_index < indices_per_block * indices_per_block * indices_per_block)
-			return block_index_from_indirect_with_allocation(m_inode_info.block[TmpInodeInfo::direct_block_count + 2], data_block_index, 3);
+			return block_index_from_indirect_with_allocation_no_lock(m_inode_info.block[TmpInodeInfo::direct_block_count + 2], data_block_index, 3);
 
 		ASSERT_NOT_REACHED();
 	}
 
-	BAN::ErrorOr<size_t> TmpInode::block_index_from_indirect_with_allocation(size_t& block, size_t index, uint32_t depth)
+	BAN::ErrorOr<size_t> TmpInode::block_index_from_indirect_with_allocation_no_lock(size_t& block, size_t index, uint32_t depth)
 	{
 		if (block == 0)
 		{
@@ -280,7 +288,7 @@ namespace Kernel
 		if (depth == 1)
 			return next_block;
 
-		return block_index_from_indirect_with_allocation(next_block, index, depth - 1);
+		return block_index_from_indirect_with_allocation_no_lock(next_block, index, depth - 1);
 	}
 
 	/* FILE INODE */
@@ -309,6 +317,8 @@ namespace Kernel
 
 	BAN::ErrorOr<size_t> TmpFileInode::read_impl(off_t offset, BAN::ByteSpan out_buffer)
 	{
+		LockGuard _(m_lock);
+
 		if (offset >= size() || out_buffer.size() == 0)
 			return 0;
 
@@ -339,7 +349,12 @@ namespace Kernel
 
 	BAN::ErrorOr<size_t> TmpFileInode::write_impl(off_t offset, BAN::ConstByteSpan in_buffer)
 	{
-		// FIXME: handle overflow
+		if (offset < 0)
+			return BAN::Error::from_errno(EINVAL);
+		if (BAN::Math::will_addition_overflow<size_t>(offset, in_buffer.size()))
+			return BAN::Error::from_errno(EOVERFLOW);
+
+		LockGuard _(m_lock);
 
 		if (offset + in_buffer.size() > (size_t)size())
 			TRY(truncate_impl(offset + in_buffer.size()));
@@ -370,6 +385,7 @@ namespace Kernel
 	{
 		// FIXME: if size is decreased, we should probably free
 		//        unused blocks
+		// FIXME: make this atomic
 
 		m_inode_info.size = new_size;
 		return {};
@@ -427,6 +443,8 @@ namespace Kernel
 
 	BAN::ErrorOr<void> TmpSymlinkInode::set_link_target_impl(BAN::StringView new_target)
 	{
+		LockGuard _(m_lock);
+
 		free_all_blocks();
 		m_inode_info.size = 0;
 
@@ -455,6 +473,8 @@ namespace Kernel
 
 	BAN::ErrorOr<BAN::String> TmpSymlinkInode::link_target_impl()
 	{
+		LockGuard _(m_lock);
+
 		BAN::String result;
 		TRY(result.resize(size()));
 
@@ -522,7 +542,7 @@ namespace Kernel
 	{
 	}
 
-	BAN::ErrorOr<void> TmpDirectoryInode::prepare_unlink()
+	BAN::ErrorOr<void> TmpDirectoryInode::prepare_unlink_no_lock()
 	{
 		ino_t dot_ino = 0;
 		ino_t dotdot_ino = 0;
@@ -564,15 +584,15 @@ namespace Kernel
 
 	BAN::ErrorOr<BAN::RefPtr<Inode>> TmpDirectoryInode::find_inode_impl(BAN::StringView name)
 	{
-		ino_t result = 0;
+		LockGuard _(m_lock);
 
+		ino_t result = 0;
 		for_each_valid_entry([&](TmpDirectoryEntry& entry) {
 			if (entry.name_sv() != name)
 				return BAN::Iteration::Continue;
 			result = entry.ino;
 			return BAN::Iteration::Break;
 		});
-
 		if (result == 0)
 			return BAN::Error::from_errno(ENOENT);
 
@@ -588,6 +608,8 @@ namespace Kernel
 			return BAN::Error::from_errno(ENOBUFS);
 		}
 
+		LockGuard _(m_lock);
+
 		auto block_index = this->block_index(data_block_index);
 
 		// if we reach a non-allocated block, it marks the end
@@ -664,6 +686,8 @@ namespace Kernel
 		ASSERT(!inode->mode().ifdir());
 		ASSERT(&m_fs == inode->filesystem());
 
+		LockGuard _(m_lock);
+
 		if (!find_inode_impl(name).is_error())
 			return BAN::Error::from_errno(EEXIST);
 
@@ -680,16 +704,17 @@ namespace Kernel
 
 		auto* tmp_parent = static_cast<TmpDirectoryInode*>(old_parent.ptr());
 
-		// FIXME: possible deadlock :)
-		LockGuard _(tmp_parent->m_mutex);
+		// FIXME: is this a possible deadlock?
+		LockGuard _0(tmp_parent->m_lock);
+		LockGuard _1(m_lock);
 
 		auto old_inode = TRY(tmp_parent->find_inode_impl(old_name));
 		auto* tmp_inode = static_cast<TmpInode*>(old_inode.ptr());
 
-		if (auto replace_or_error = find_inode_impl(new_name); replace_or_error.is_error())
+		if (auto find_result = find_inode_impl(new_name); find_result.is_error())
 		{
-			if (replace_or_error.error().get_error_code() != ENOENT)
-				return replace_or_error.release_error();
+			if (find_result.error().get_error_code() != ENOENT)
+				return find_result.release_error();
 		}
 		else
 		{
@@ -711,15 +736,15 @@ namespace Kernel
 
 	BAN::ErrorOr<void> TmpDirectoryInode::unlink_inode(BAN::StringView name, bool cleanup)
 	{
-		ino_t entry_ino = 0;
+		LockGuard _(m_lock);
 
+		ino_t entry_ino = 0;
 		for_each_valid_entry([&](TmpDirectoryEntry& entry) {
 			if (entry.name_sv() != name)
 				return BAN::Iteration::Continue;
 			entry_ino = entry.ino;
 			return BAN::Iteration::Break;
 		});
-
 		if (entry_ino == 0)
 			return BAN::Error::from_errno(ENOENT);
 
@@ -728,7 +753,7 @@ namespace Kernel
 		ASSERT(inode->nlink() > 0);
 
 		if (cleanup)
-			TRY(inode->prepare_unlink());
+			TRY(inode->prepare_unlink_no_lock());
 		inode->m_inode_info.nlink--;
 
 		if (inode->nlink() == 0)
@@ -749,6 +774,8 @@ namespace Kernel
 	{
 		static constexpr size_t directory_entry_alignment = sizeof(TmpDirectoryEntry);
 
+		LockGuard _(m_lock);
+
 		auto find_result = find_inode_impl(name);
 		if (!find_result.is_error())
 			return BAN::Error::from_errno(EEXIST);

kernel/kernel/FS/USTARModule.cpp

@@ -1,5 +1,6 @@
 #include <BAN/ScopeGuard.h>
 #include <kernel/FS/USTARModule.h>
+#include <kernel/Memory/PageTable.h>
 #include <kernel/Timer/Timer.h>
 #include <LibDEFLATE/Decompressor.h>
 

kernel/kernel/Input/InputDevice.cpp

@@ -13,12 +13,15 @@
 namespace Kernel
 {
 
+	static SpinLock s_keyboard_lock;
 	static BAN::Vector<BAN::WeakPtr<InputDevice>> s_keyboards;
 	static BAN::RefPtr<KeyboardDevice> s_keyboard_device;
 
+	static SpinLock s_mouse_lock;
 	static BAN::Vector<BAN::WeakPtr<InputDevice>> s_mice;
 	static BAN::RefPtr<MouseDevice> s_mouse_device;
 
+	static SpinLock s_joystick_lock;
 	static BAN::Vector<BAN::WeakPtr<InputDevice>> s_joysticks;
 
 	static const char* get_name_format(InputDevice::Type type)
@@ -40,20 +43,29 @@ namespace Kernel
 		switch (type)
 		{
 			case InputDevice::Type::Keyboard:
+			{
+				SpinLockGuard _(s_keyboard_lock);
 				for (size_t i = 0; i < s_keyboards.size(); i++)
 					if (!s_keyboards[i].valid())
 						return makedev(DeviceNumber::Keyboard, i + 1);
 				return makedev(DeviceNumber::Keyboard, s_keyboards.size() + 1);
+			}
 			case InputDevice::Type::Mouse:
+			{
+				SpinLockGuard _(s_mouse_lock);
 				for (size_t i = 0; i < s_mice.size(); i++)
 					if (!s_mice[i].valid())
 						return makedev(DeviceNumber::Mouse, i + 1);
 				return makedev(DeviceNumber::Mouse, s_mice.size() + 1);
+			}
 			case InputDevice::Type::Joystick:
+			{
+				SpinLockGuard _(s_joystick_lock);
 				for (size_t i = 0; i < s_joysticks.size(); i++)
 					if (!s_joysticks[i].valid())
 						return makedev(DeviceNumber::Joystick, i + 1);
 				return makedev(DeviceNumber::Joystick, s_joysticks.size() + 1);
+			}
 		}
 		ASSERT_NOT_REACHED();
 	}
@@ -84,20 +96,29 @@ namespace Kernel
 		switch (m_type)
 		{
 			case Type::Keyboard:
+			{
+				SpinLockGuard _(s_keyboard_lock);
 				if (s_keyboards.size() < minor(m_rdev))
 					MUST(s_keyboards.resize(minor(m_rdev)));
 				s_keyboards[minor(m_rdev) - 1] = MUST(get_weak_ptr());
 				break;
+			}
 			case Type::Mouse:
+			{
+				SpinLockGuard _(s_mouse_lock);
 				if (s_mice.size() < minor(m_rdev))
 					MUST(s_mice.resize(minor(m_rdev)));
 				s_mice[minor(m_rdev) - 1] = MUST(get_weak_ptr());
 				break;
+			}
 			case Type::Joystick:
+			{
+				SpinLockGuard _(s_joystick_lock);
 				if (s_joysticks.size() < minor(m_rdev))
 					MUST(s_joysticks.resize(minor(m_rdev)));
 				s_joysticks[minor(m_rdev) - 1] = MUST(get_weak_ptr());
 				break;
+			}
 		}
 	}
 
@@ -256,6 +277,8 @@ namespace Kernel
 	void KeyboardDevice::notify()
 	{
 		epoll_notify(EPOLLIN);
+
+		SpinLockGuard _(s_keyboard_lock);
 		m_thread_blocker.unblock();
 	}
 
@@ -264,6 +287,7 @@ namespace Kernel
 		if (buffer.size() < sizeof(LibInput::RawKeyEvent))
 			return BAN::Error::from_errno(ENOBUFS);
 
+		SpinLockGuard keyboard_guard(s_keyboard_lock);
 		for (;;)
 		{
 			for (auto& weak_keyboard : s_keyboards)
@@ -277,13 +301,14 @@ namespace Kernel
 					return bytes;
 			}
 
-			// FIXME: race condition as notify doesn't lock mutex
-			TRY(Thread::current().block_or_eintr_indefinite(m_thread_blocker, &m_mutex));
+			SpinLockGuardAsMutex smutex(keyboard_guard);
+			TRY(Thread::current().block_or_eintr_indefinite(m_thread_blocker, &smutex));
 		}
 	}
 
 	bool KeyboardDevice::can_read_impl() const
 	{
+		SpinLockGuard _(s_keyboard_lock);
 		for (auto& weak_keyboard : s_keyboards)
 			if (auto keyboard = weak_keyboard.lock())
 				if (keyboard->can_read())
@@ -308,6 +333,8 @@ namespace Kernel
 	void MouseDevice::notify()
 	{
 		epoll_notify(EPOLLIN);
+
+		SpinLockGuard _(s_mouse_lock);
 		m_thread_blocker.unblock();
 	}
 
@@ -316,6 +343,7 @@ namespace Kernel
 		if (buffer.size() < sizeof(LibInput::MouseEvent))
 			return BAN::Error::from_errno(ENOBUFS);
 
+		SpinLockGuard mouse_guard(s_mouse_lock);
 		for (;;)
 		{
 			for (auto& weak_mouse : s_mice)
@@ -329,13 +357,14 @@ namespace Kernel
 					return bytes;
 			}
 
-			// FIXME: race condition as notify doesn't lock mutex
-			TRY(Thread::current().block_or_eintr_indefinite(m_thread_blocker, &m_mutex));
+			SpinLockGuardAsMutex smutex(mouse_guard);
+			TRY(Thread::current().block_or_eintr_indefinite(m_thread_blocker, &smutex));
 		}
 	}
 
 	bool MouseDevice::can_read_impl() const
 	{
+		SpinLockGuard _(s_mouse_lock);
 		for (auto& weak_mouse : s_mice)
 			if (auto mouse = weak_mouse.lock())
 				if (mouse->can_read())

kernel/kernel/Memory/ByteRingBuffer.cpp

@@ -25,10 +25,12 @@ namespace Kernel
 			const paddr_t paddr = Heap::get().take_free_page();
 			if (paddr == 0)
 				return BAN::Error::from_errno(ENOMEM);
-			PageTable::kernel().map_page_at(paddr, buffer->m_vaddr        + i * PAGE_SIZE, PageTable::ReadWrite | PageTable::Present);
-			PageTable::kernel().map_page_at(paddr, buffer->m_vaddr + size + i * PAGE_SIZE, PageTable::ReadWrite | PageTable::Present);
+			PageTable::kernel().map_page_at(paddr, buffer->m_vaddr        + i * PAGE_SIZE, PageTable::ReadWrite | PageTable::Present, PageTable::MemoryType::Normal, false);
+			PageTable::kernel().map_page_at(paddr, buffer->m_vaddr + size + i * PAGE_SIZE, PageTable::ReadWrite | PageTable::Present, PageTable::MemoryType::Normal, false);
 		}
 
+		PageTable::kernel().invalidate_range(buffer->m_vaddr, page_count * 2, true);
+
 		return buffer;
 	}
 
@@ -36,13 +38,11 @@ namespace Kernel
 	{
 		if (m_vaddr == 0)
 			return;
+
 		for (size_t i = 0; i < m_capacity / PAGE_SIZE; i++)
-		{
-			const paddr_t paddr = PageTable::kernel().physical_address_of(m_vaddr + i * PAGE_SIZE);
-			if (paddr == 0)
-				break;
-			Heap::get().release_page(paddr);
-		}
+			if (const paddr_t paddr = PageTable::kernel().physical_address_of(m_vaddr + i * PAGE_SIZE))
+				Heap::get().release_page(paddr);
+
 		PageTable::kernel().unmap_range(m_vaddr, m_capacity * 2);
 	}
 

kernel/kernel/Memory/FileBackedRegion.cpp

@@ -2,8 +2,12 @@
 #include <kernel/Memory/FileBackedRegion.h>
 #include <kernel/Memory/Heap.h>
 
+#include <BAN/ScopeGuard.h>
+
 #include <sys/mman.h>
 
+#pragma GCC diagnostic ignored "-Wstack-usage="
+
 namespace Kernel
 {
 
@@ -31,7 +35,7 @@ namespace Kernel
 		if (type == Type::PRIVATE)
 			TRY(region->m_dirty_pages.resize(BAN::Math::div_round_up<size_t>(size, PAGE_SIZE)));
 
-		LockGuard _(inode->m_mutex);
+		SpinLockGuard _(inode->m_shared_region_lock);
 		if (!(region->m_shared_data = inode->m_shared_region.lock()))
 		{
 			auto shared_data = TRY(BAN::RefPtr<SharedFileData>::create());
@@ -62,28 +66,23 @@ namespace Kernel
 
 	SharedFileData::~SharedFileData()
 	{
-		// no-one should be referencing this anymore
-		[[maybe_unused]] bool success = mutex.try_lock();
-		ASSERT(success);
+		// TODO: validate that this is not locked
 
 		for (size_t i = 0; i < pages.size(); i++)
 		{
 			if (pages[i] == 0)
 				continue;
-			sync(i);
+			sync_no_lock(i);
 			Heap::get().release_page(pages[i]);
 		}
-
-		mutex.unlock();
 	}
 
-	void SharedFileData::sync(size_t page_index)
+	void SharedFileData::sync_no_lock(size_t page_index)
 	{
-		ASSERT(mutex.is_locked());
-
 		if (pages[page_index] == 0)
 			return;
 
+		uint8_t page_buffer[PAGE_SIZE];
 		PageTable::with_fast_page(pages[page_index], [&] {
 			memcpy(page_buffer, PageTable::fast_page_as_ptr(), PAGE_SIZE);
 		});
@@ -100,13 +99,12 @@ namespace Kernel
 		if (m_type != Type::SHARED)
 			return {};
 
-		const vaddr_t first_page = address & PAGE_ADDR_MASK;
-		const vaddr_t last_page  = BAN::Math::div_round_up<vaddr_t>(address + size, PAGE_SIZE) * PAGE_SIZE;
+		const vaddr_t first_page = BAN::Math::max(m_vaddr, address) & PAGE_ADDR_MASK;
+		const vaddr_t last_page = BAN::Math::div_round_up(BAN::Math::min(m_vaddr + m_size, address + size), PAGE_SIZE) * PAGE_SIZE;
 
-		LockGuard _(m_shared_data->mutex);
+		RWLockRDGuard _(m_shared_data->rw_lock);
 		for (vaddr_t page_addr = first_page; page_addr < last_page; page_addr += PAGE_SIZE)
-			if (contains(page_addr))
-				m_shared_data->sync((page_addr - m_vaddr) / PAGE_SIZE);
+			m_shared_data->sync_no_lock((m_offset + page_addr - m_vaddr) / PAGE_SIZE);
 
 		return {};
 	}
@@ -125,29 +123,43 @@ namespace Kernel
 		if (m_page_table.physical_address_of(vaddr) == 0)
 		{
 			ASSERT(m_shared_data);
-			LockGuard _(m_shared_data->mutex);
+
+			uint8_t page_buffer[PAGE_SIZE];
+
+			m_shared_data->rw_lock.rd_lock();
 
 			bool shared_data_has_correct_page = false;
 			if (m_shared_data->pages[shared_page_index] == 0)
 			{
-				m_shared_data->pages[shared_page_index] = Heap::get().take_free_page();
-				if (m_shared_data->pages[shared_page_index] == 0)
-					return BAN::Error::from_errno(ENOMEM);
+				m_shared_data->rw_lock.rd_unlock();
 
 				const size_t offset = (vaddr - m_vaddr) + m_offset;
-				ASSERT(offset % 4096 == 0);
+				ASSERT(offset % PAGE_SIZE == 0);
 
 				const size_t bytes = BAN::Math::min<size_t>(m_inode->size() - offset, PAGE_SIZE);
 
-				memset(m_shared_data->page_buffer, 0x00, PAGE_SIZE);
-				TRY(m_inode->read(offset, BAN::ByteSpan(m_shared_data->page_buffer, bytes)));
-				shared_data_has_correct_page = true;
+				TRY(m_inode->read(offset, BAN::ByteSpan(page_buffer, bytes)));
+				memset(page_buffer + bytes, 0, PAGE_SIZE - bytes);
 
-				PageTable::with_fast_page(m_shared_data->pages[shared_page_index], [&] {
-					memcpy(PageTable::fast_page_as_ptr(), m_shared_data->page_buffer, PAGE_SIZE);
-				});
+				{
+					RWLockWRGuard _(m_shared_data->rw_lock);
+					if (m_shared_data->pages[shared_page_index] == 0)
+					{
+						m_shared_data->pages[shared_page_index] = Heap::get().take_free_page();
+						if (m_shared_data->pages[shared_page_index] == 0)
+							return BAN::Error::from_errno(ENOMEM);
+						PageTable::with_fast_page(m_shared_data->pages[shared_page_index], [&] {
+							memcpy(PageTable::fast_page_as_ptr(), page_buffer, PAGE_SIZE);
+						});
+						shared_data_has_correct_page = true;
+					}
+				}
+
+				m_shared_data->rw_lock.rd_lock();
 			}
 
+			BAN::ScopeGuard _([this] { m_shared_data->rw_lock.rd_unlock(); });
+
 			if (m_type == Type::PRIVATE && wants_write)
 			{
 				const paddr_t paddr = Heap::get().take_free_page();
@@ -156,11 +168,11 @@ namespace Kernel
 				if (!shared_data_has_correct_page)
 				{
 					PageTable::with_fast_page(m_shared_data->pages[shared_page_index], [&] {
-						memcpy(m_shared_data->page_buffer, PageTable::fast_page_as_ptr(), PAGE_SIZE);
+						memcpy(page_buffer, PageTable::fast_page_as_ptr(), PAGE_SIZE);
 					});
 				}
 				PageTable::with_fast_page(paddr, [&] {
-					memcpy(PageTable::fast_page_as_ptr(), m_shared_data->page_buffer, PAGE_SIZE);
+					memcpy(PageTable::fast_page_as_ptr(), page_buffer, PAGE_SIZE);
 				});
 				m_dirty_pages[local_page_index] = paddr;
 				m_page_table.map_page_at(paddr, vaddr, m_flags);
@@ -176,29 +188,17 @@ namespace Kernel
 		}
 		else
 		{
-			// page does not need remappings
-			if (m_type != Type::PRIVATE || !wants_write)
-				return false;
-			ASSERT(writable());
-
-			// page is already mapped as writable
-			if (m_page_table.get_page_flags(vaddr) & PageTable::Flags::ReadWrite)
-				return false;
+			ASSERT(m_type == Type::PRIVATE && wants_write);
 
 			const paddr_t paddr = Heap::get().take_free_page();
 			if (paddr == 0)
 				return BAN::Error::from_errno(ENOMEM);
 
-			ASSERT(m_shared_data);
-			LockGuard _(m_shared_data->mutex);
-			ASSERT(m_shared_data->pages[shared_page_index]);
+			ASSERT(&m_page_table == &PageTable::current());
+			PageTable::with_fast_page(paddr, [vaddr] {
+				memcpy(PageTable::fast_page_as_ptr(), reinterpret_cast<void*>(vaddr), PAGE_SIZE);
+			});
 
-			PageTable::with_fast_page(m_shared_data->pages[shared_page_index], [&] {
-				memcpy(m_shared_data->page_buffer, PageTable::fast_page_as_ptr(), PAGE_SIZE);
-			});
-			PageTable::with_fast_page(paddr, [&] {
-				memcpy(PageTable::fast_page_as_ptr(), m_shared_data->page_buffer, PAGE_SIZE);
-			});
 			m_dirty_pages[local_page_index] = paddr;
 			m_page_table.map_page_at(paddr, vaddr, m_flags);
 		}

kernel/kernel/Memory/VirtualRange.cpp

@@ -68,8 +68,9 @@ namespace Kernel
 			PageTable::with_fast_page(paddr, [] {
 				memset(PageTable::fast_page_as_ptr(), 0, PAGE_SIZE);
 			});
-			m_page_table.map_page_at(paddr, vaddr() + i * PAGE_SIZE, m_flags);
+			m_page_table.map_page_at(paddr, vaddr() + i * PAGE_SIZE, m_flags, PageTable::MemoryType::Normal, false);
 		}
+		m_page_table.invalidate_range(m_vaddr, page_count, true);
 		return {};
 	}
 

kernel/kernel/Memory/kmalloc.cpp

@@ -1,438 +1,292 @@
-#include <BAN/Errors.h>
-#include <kernel/BootInfo.h>
+#include <kernel/Memory/Heap.h>
 #include <kernel/Memory/kmalloc.h>
+#include <kernel/Memory/PageTable.h>
 
-#define MB (1 << 20)
+static constexpr size_t s_allocator_chunk_size { 64 };
+static constexpr size_t s_allocator_align      { alignof(max_align_t) };
 
-extern uint8_t g_kernel_end[];
+static constexpr size_t s_max_allocator_count { 128 };
 
-static constexpr size_t s_kmalloc_min_align = alignof(max_align_t);
+static constexpr size_t s_allocator_default_size {       128 * 1024 };
+static constexpr size_t s_allocator_dynamic_size { 16 * 1024 * 1024 };
 
-static constexpr size_t s_kmalloc_size = 48 * MB;
-static constexpr size_t s_kmalloc_fixed_size = 16 * MB;
-static uint8_t s_kmalloc_storage[s_kmalloc_size + s_kmalloc_fixed_size];
+alignas(s_allocator_align) static uint8_t s_default_allocator_memory[s_allocator_default_size] {};
 
-struct kmalloc_node
+// NOTE: 128 KiB + 127 * 16 MiB ~= 2 GiB
+//       This is should be more than enough for kmalloc :^)
+
+struct BitmapAllocator
 {
-	void set_align(ptrdiff_t align)			{ m_align = align; }
-	void set_end(uintptr_t end)				{ m_size = end - (uintptr_t)m_data; }
-	void set_used(bool used)				{ m_used = used; }
-
-	bool can_align(uint32_t align)			{ return align < m_size; }
-	bool can_fit_before()					{ return m_align > sizeof(kmalloc_node); }
-	bool can_fit_after(size_t new_size)		{ return data() + new_size < end() - sizeof(kmalloc_node); }
-
-	void split_in_align()
+	struct Header
 	{
-		uintptr_t node_end = end();
-		set_end(data() - sizeof(kmalloc_node));
-		set_align(0);
+		size_t chunks { 0 };
+		uint8_t padding[s_allocator_align - sizeof(chunks)];
+	};
 
-		auto* next = after();
-		next->set_end(node_end);
-		next->set_align(0);
+	uint32_t bitmap_chunks { 0 };
+	uint32_t total_chunks  { 0 };
+	uint32_t free_chunks   { 0 };
+	uint32_t allocations   { 0 };
+	uint8_t* base          { nullptr };
+
+	static size_t needed_chunks(size_t size)
+	{
+		return BAN::Math::div_round_up(sizeof(BitmapAllocator::Header) + size, s_allocator_chunk_size);
 	}
 
-	void split_after_size(size_t size)
+	void initialize_default()
 	{
-		uintptr_t node_end = end();
-		set_end(data() + size);
+		constexpr size_t bitmap_bytes  = BAN::Math::div_round_up(s_allocator_default_size, s_allocator_chunk_size * 8);
+		constexpr size_t bitmap_chunks = BAN::Math::div_round_up(bitmap_bytes, s_allocator_chunk_size);
+		constexpr size_t usable_chunks = s_allocator_default_size / s_allocator_chunk_size - bitmap_chunks;
 
-		auto* next = after();
-		next->set_end(node_end);
-		next->set_align(0);
+		this->bitmap_chunks = bitmap_chunks;
+		this->total_chunks  = usable_chunks;
+		this->free_chunks   = usable_chunks;
+		this->base          = s_default_allocator_memory;
+
+		memset(this->base, 0, bitmap_chunks * s_allocator_chunk_size);
 	}
 
-	bool used()					{ return m_used; }
-	uintptr_t size_no_align()	{ return m_size; }
-	uintptr_t size()			{ return size_no_align() - m_align; }
-	uintptr_t data_no_align()	{ return (uintptr_t)m_data; }
-	uintptr_t data()			{ return data_no_align() + m_align; }
-	uintptr_t end()				{ return data_no_align() + m_size; }
-	kmalloc_node* after()		{ return (kmalloc_node*)end(); }
-
-private:
-	uint32_t	m_size;
-	uint32_t	m_align;
-	bool		m_used;
-	uint8_t 	m_padding[s_kmalloc_min_align - sizeof(m_size) - sizeof(m_align) - sizeof(m_used)];
-	uint8_t		m_data[0];
-};
-static_assert(sizeof(kmalloc_node) == s_kmalloc_min_align);
-
-struct kmalloc_info
-{
-	const uintptr_t	base = (uintptr_t)s_kmalloc_storage;
-	const size_t	size = s_kmalloc_size;
-	const uintptr_t	end = base + size;
-
-	kmalloc_node* first() { return (kmalloc_node*)base; }
-	kmalloc_node* from_address(void* addr)
+	bool initialize_dynamic()
 	{
-		for (auto* node = first(); node->end() < end; node = node->after())
-			if (node->data() == (uintptr_t)addr)
-				return node;
+		using namespace Kernel;
+
+		const size_t page_count = s_allocator_dynamic_size / PAGE_SIZE;
+
+		const vaddr_t vaddr = PageTable::kernel().reserve_free_contiguous_pages(page_count, KERNEL_OFFSET);
+		if (vaddr == 0)
+			return false;
+
+		for (size_t i = 0; i < page_count; i++)
+		{
+			const paddr_t paddr = Heap::get().take_free_page();
+			if (paddr == 0)
+			{
+				for (size_t j = 0; j < i; j++)
+					Heap::get().release_page(PageTable::kernel().physical_address_of(vaddr + j * PAGE_SIZE));
+				PageTable::kernel().unmap_range(vaddr, page_count * PAGE_SIZE);
+				return false;
+			}
+
+			PageTable::kernel().map_page_at(paddr, vaddr + i * PAGE_SIZE, PageTable::ReadWrite | PageTable::Present);
+		}
+
+		constexpr size_t bitmap_bytes  = BAN::Math::div_round_up(s_allocator_dynamic_size, s_allocator_chunk_size * 8);
+		constexpr size_t bitmap_chunks = BAN::Math::div_round_up(bitmap_bytes, s_allocator_chunk_size);
+		constexpr size_t usable_chunks = s_allocator_dynamic_size / s_allocator_chunk_size - bitmap_chunks;
+
+		this->bitmap_chunks = bitmap_chunks;
+		this->total_chunks  = usable_chunks;
+		this->free_chunks   = usable_chunks;
+		this->base          = reinterpret_cast<uint8_t*>(vaddr);
+
+		memset(this->base, 0, bitmap_chunks * s_allocator_chunk_size);
+
+		return true;
+	}
+
+	uint8_t* data_start() { return base + bitmap_chunks * s_allocator_chunk_size; }
+	const uint8_t* data_start() const { return base + bitmap_chunks * s_allocator_chunk_size; }
+
+	size_t get_first_chunk(void* ptr) const
+	{
+		return (static_cast<uint8_t*>(ptr) - sizeof(Header) - data_start()) / s_allocator_chunk_size;
+	}
+
+	bool contains(void* ptr) const
+	{
+		if (ptr < data_start() + sizeof(Header))
+			return false;
+		return get_first_chunk(ptr) < total_chunks;
+	}
+
+	bool get_bit(size_t index) const
+	{
+		ASSERT(index < total_chunks);
+		const size_t byte = index / 8;
+		const size_t bit = index % 8;
+		return (base[byte] >> bit) & 1;
+	}
+
+	void set_bit(size_t index, bool value)
+	{
+		ASSERT(index < total_chunks);
+		const size_t byte = index / 8;
+		const size_t bit = index % 8;
+		if (value)
+			base[byte] |= 1 << bit;
+		else
+			base[byte] &= ~(1 << bit);
+	}
+
+	size_t find_unset_bit(size_t index) const
+	{
+		// NOTE: We could optimize other bitmap functions than this
+		//       but this one is the bottle neck so it doesn't matter
+
+		static_assert(sizeof(unsigned long long) == sizeof(uint64_t));
+
+		if (index >= total_chunks)
+			return index;
+
+		if (const auto rem = index % 64)
+		{
+			const uint64_t qword = *reinterpret_cast<const uint64_t*>(base + (index - rem) / 8) >> rem;
+			if (qword != (1ull << (64 - rem)) - 1)
+				return index + __builtin_ctzll(~qword);
+			index += 64 - rem;
+		}
+
+		while (index < total_chunks)
+		{
+			const uint64_t qword = *reinterpret_cast<const uint64_t*>(base + index / 8);
+			if (qword != UINT64_MAX)
+				return index + __builtin_ctzll(~qword);
+			index += 64;
+		}
+
+		return index;
+	}
+
+	size_t count_unset_bits(size_t index, size_t wanted) const
+	{
+		size_t count = 0;
+		for (; index + count < total_chunks && count < wanted; count++)
+			if (get_bit(index + count))
+				break;
+		return count;
+	}
+
+	Header& header_from_chunk(size_t index)
+	{
+		return *reinterpret_cast<Header*>(data_start() + index * s_allocator_chunk_size);
+	}
+
+	Header& header_from_ptr(void* ptr)
+	{
+		return *reinterpret_cast<Header*>(static_cast<uint8_t*>(ptr) - sizeof(Header));
+	}
+
+	void* allocate(size_t needed_chunks)
+	{
+		ASSERT(needed_chunks > 0);
+
+		if (needed_chunks > free_chunks)
+			return nullptr;
+
+		for (size_t i = find_unset_bit(0); i <= total_chunks - needed_chunks; i = find_unset_bit(i))
+		{
+			if (const size_t count = count_unset_bits(i, needed_chunks); count < needed_chunks)
+			{
+				i += count + 1;
+				continue;
+			}
+
+			for (size_t j = 0; j < needed_chunks; j++)
+				set_bit(i + j, true);
+
+			auto& header = header_from_chunk(i);
+			header.chunks = needed_chunks;
+
+			free_chunks -= header.chunks;
+			allocations++;
+
+			return &header + 1;
+		}
+
 		return nullptr;
 	}
 
-	bool contains(uintptr_t addr) const
+	void free(void* ptr)
 	{
-		return base <= addr && addr < end;
-	}
+		ASSERT(contains(ptr));
 
-	size_t used = 0;
-	size_t free = size;
+		const size_t first_chunk = get_first_chunk(ptr);
+
+		auto& header = header_from_ptr(ptr);
+		for (size_t i = 0; i < header.chunks; i++)
+			set_bit(first_chunk + i, false);
+
+		free_chunks += header.chunks;
+		allocations--;
+	}
 };
-static kmalloc_info s_kmalloc_info;
+
+static uint8_t s_allocator_storage[s_max_allocator_count * sizeof(BitmapAllocator)];
+static BitmapAllocator* s_allocators[s_max_allocator_count] {};
 
 static Kernel::SpinLock s_kmalloc_lock;
 
-template<size_t SIZE>
-struct kmalloc_fixed_node
-{
-	uint8_t data[SIZE - 2 * sizeof(uint16_t)];
-	uint16_t prev = NULL;
-	uint16_t next = NULL;
-	static constexpr uint16_t invalid = ~0;
-};
-
-struct kmalloc_fixed_info
-{
-	using node = kmalloc_fixed_node<64>;
-
-	const uintptr_t	base = s_kmalloc_info.end;
-	const size_t	size = s_kmalloc_fixed_size;
-	const uintptr_t	end = base + size;
-	const size_t	node_count = size / sizeof(node);
-
-	node* free_list_head = NULL;
-	node* used_list_head = NULL;
-
-	node* node_at(size_t index) { return (node*)(base + index * sizeof(node)); }
-	uint16_t index_of(const node* p) { return ((uintptr_t)p - base) / sizeof(node); }
-
-	size_t used = 0;
-	size_t free = size;
-};
-static kmalloc_fixed_info s_kmalloc_fixed_info;
-
 void kmalloc_initialize()
 {
-	dprintln("kmalloc {8H}->{8H}", (uintptr_t)s_kmalloc_storage, (uintptr_t)s_kmalloc_storage + sizeof(s_kmalloc_storage));
-
-	// initialize fixed size allocations
-	{
-		auto& info = s_kmalloc_fixed_info;
-
-		for (size_t i = 0; i < info.node_count; i++)
-		{
-			auto* node = info.node_at(i);
-			node->next = i - 1;
-			node->prev = i + 1;
-		}
-
-		info.node_at(0)->next = kmalloc_fixed_info::node::invalid;
-		info.node_at(info.node_count - 1)->prev = kmalloc_fixed_info::node::invalid;
-
-		info.free_list_head = info.node_at(0);
-		info.used_list_head = nullptr;
-	}
-
-	// initial general allocations
-	{
-		auto& info = s_kmalloc_info;
-		auto* node = info.first();
-		node->set_end(info.end);
-		node->set_align(0);
-		node->set_used(false);
-	}
+	auto& allocator = reinterpret_cast<BitmapAllocator*>(s_allocator_storage)[0];
+	new (&allocator) BitmapAllocator();
+	allocator.initialize_default();
+	s_allocators[0] = &allocator;
 }
 
-void kmalloc_dump_info()
+static void kmalloc_dump_info()
 {
-	Kernel::SpinLockGuard _(s_kmalloc_lock);
+	ASSERT(s_kmalloc_lock.current_processor_has_lock());
 
-	dprintln("kmalloc:               0x{8H}->0x{8H}", s_kmalloc_info.base, s_kmalloc_info.end);
-	dprintln("  used: 0x{8H}", s_kmalloc_info.used);
-	dprintln("  free: 0x{8H}", s_kmalloc_info.free);
-
-	dprintln("kmalloc fixed {} byte: 0x{8H}->0x{8H}", sizeof(kmalloc_fixed_info::node), s_kmalloc_fixed_info.base, s_kmalloc_fixed_info.end);
-	dprintln("  used: 0x{8H}", s_kmalloc_fixed_info.used);
-	dprintln("  free: 0x{8H}", s_kmalloc_fixed_info.free);
-}
-
-static bool is_corrupted()
-{
-	Kernel::SpinLockGuard _(s_kmalloc_lock);
-	auto& info = s_kmalloc_info;
-	auto* temp = info.first();
-	while (reinterpret_cast<uintptr_t>(temp) != info.end)
+	dwarnln("kmalloc info");
+	for (size_t i = 0; i < s_max_allocator_count && s_allocators[i]; i++)
 	{
-		if (!info.contains(reinterpret_cast<uintptr_t>(temp)))
-			return true;
-		if (!info.contains(temp->end() - 1))
-			return true;
-		if (temp->after() <= temp)
-			return true;
-		temp = temp->after();
+		dwarnln("  allocator {}", i);
+		dwarnln("    total size:  {}", s_allocators[i]->total_chunks * s_allocator_chunk_size);
+		dwarnln("    free size:   {}", s_allocators[i]->free_chunks  * s_allocator_chunk_size);
+		dwarnln("    allocations: {}", s_allocators[i]->allocations);
 	}
-	return false;
-}
-
-[[maybe_unused]] static void debug_dump()
-{
-	Kernel::SpinLockGuard _(s_kmalloc_lock);
-
-	auto& info = s_kmalloc_info;
-
-	uint32_t used = 0;
-	uint32_t free = 0;
-
-	for (auto* node = info.first(); node->data() <= info.end; node = node->after())
-	{
-		(node->used() ? used : free) += sizeof(kmalloc_node) + node->size_no_align();
-		dprintln("{} node {H} -> {H}", node->used() ? "used" : "free", node->data(), node->end());
-	}
-
-	dprintln("total used: {}", used);
-	dprintln("total free: {}", free);
-	dprintln("            {}", used + free);
-}
-
-static void* kmalloc_fixed()
-{
-	Kernel::SpinLockGuard _(s_kmalloc_lock);
-
-	auto& info = s_kmalloc_fixed_info;
-
-	if (!info.free_list_head)
-		return nullptr;
-
-	// allocate the node on top of free list
-	auto* node = info.free_list_head;
-	ASSERT(node->next == kmalloc_fixed_info::node::invalid);
-
-	// remove the node from free list
-	if (info.free_list_head->prev != kmalloc_fixed_info::node::invalid)
-	{
-		info.free_list_head = info.node_at(info.free_list_head->prev);
-		info.free_list_head->next = kmalloc_fixed_info::node::invalid;
-	}
-	else
-	{
-		derrorln("removing free list, allocated {}", info.used);
-		info.free_list_head = nullptr;
-	}
-	node->prev = kmalloc_fixed_info::node::invalid;
-	node->next = kmalloc_fixed_info::node::invalid;
-
-	// move the node to the top of used nodes
-	if (info.used_list_head)
-	{
-		info.used_list_head->next = info.index_of(node);
-		node->prev = info.index_of(info.used_list_head);
-	}
-	info.used_list_head = node;
-
-	info.used += sizeof(kmalloc_fixed_info::node);
-	info.free -= sizeof(kmalloc_fixed_info::node);
-
-	return (void*)node->data;
-}
-
-static void* kmalloc_impl(size_t size, size_t align)
-{
-	ASSERT(align % s_kmalloc_min_align == 0);
-	ASSERT(size % s_kmalloc_min_align == 0);
-
-	Kernel::SpinLockGuard _(s_kmalloc_lock);
-
-	auto& info = s_kmalloc_info;
-
-	for (auto* node = info.first(); info.contains(reinterpret_cast<uintptr_t>(node)); node = node->after())
-	{
-		if (node->used())
-			continue;
-
-		if (auto* next = node->after(); info.contains(reinterpret_cast<uintptr_t>(next)))
-			if (!next->used())
-				node->set_end(next->end());
-
-		if (node->size_no_align() < size)
-			continue;
-
-		ptrdiff_t needed_align = 0;
-		if (ptrdiff_t rem = node->data_no_align() % align)
-			needed_align = align - rem;
-
-		if (!node->can_align(needed_align))
-			continue;
-
-		node->set_align(needed_align);
-		ASSERT(node->data() % align == 0);
-
-		if (node->size() < size)
-			continue;
-
-		if (node->can_fit_before())
-		{
-			node->split_in_align();
-			node->set_used(false);
-
-			node = node->after();
-			ASSERT(node->data() % align == 0);
-		}
-
-		node->set_used(true);
-
-		if (node->can_fit_after(size))
-		{
-			node->split_after_size(size);
-			node->after()->set_used(false);
-			ASSERT(node->data() % align == 0);
-		}
-
-		info.used += sizeof(kmalloc_node) + node->size_no_align();
-		info.free -= sizeof(kmalloc_node) + node->size_no_align();
-
-		return (void*)node->data();
-	}
-
-	return nullptr;
 }
 
 void* kmalloc(size_t size)
 {
-	return kmalloc(size, s_kmalloc_min_align, false);
-}
+	const size_t needed_chunks = BitmapAllocator::needed_chunks(size);
 
-static constexpr bool is_power_of_two(size_t value)
-{
-	if (value == 0)
-		return false;
-	return (value & (value - 1)) == 0;
-}
+	Kernel::SpinLockGuard _(s_kmalloc_lock);
 
-void* kmalloc(size_t size, size_t align, bool force_identity_map)
-{
-	// currently kmalloc is always identity mapped
-	(void)force_identity_map;
+	for (size_t i = 0; i < s_max_allocator_count; i++)
+	{
+		if (auto* allocator = s_allocators[i])
+		{
+			if (void* result = allocator->allocate(needed_chunks))
+				return result;
+			continue;
+		}
 
-	if (size == 0)
-		size = 1;
+		auto& new_allocator = reinterpret_cast<BitmapAllocator*>(s_allocator_storage)[i];
+		new (&new_allocator) BitmapAllocator();
+		if (!new_allocator.initialize_dynamic())
+		{
+			new_allocator.~BitmapAllocator();
+			break;
+		}
 
-	const kmalloc_info& info = s_kmalloc_info;
+		s_allocators[i] = &new_allocator;
 
-	ASSERT(is_power_of_two(align));
-	if (align < s_kmalloc_min_align)
-		align = s_kmalloc_min_align;
-	ASSERT(align <= PAGE_SIZE);
-
-	if (size == 0 || size >= info.size)
-		goto no_memory;
-
-	// if the size fits into fixed node, we will try to use that since it is faster
-	if (align == s_kmalloc_min_align && size <= sizeof(kmalloc_fixed_info::node::data))
-		if (void* result = kmalloc_fixed())
+		if (void* result = new_allocator.allocate(needed_chunks))
 			return result;
 
-	if (ptrdiff_t rem = size % s_kmalloc_min_align)
-		size += s_kmalloc_min_align - rem;
+		break;
+	}
 
-	if (void* res = kmalloc_impl(size, align))
-		return res;
-
-no_memory:
-	dwarnln("could not allocate {H} bytes ({} aligned)", size, align);
-	dwarnln(" {6H} free (fixed)", s_kmalloc_fixed_info.free);
-	dwarnln(" {6H} free", s_kmalloc_info.free);
-	//Debug::dump_stack_trace();
-	ASSERT(!is_corrupted());
+	dwarnln("failed to allocate {} bytes", size);
+	kmalloc_dump_info();
 
 	return nullptr;
 }
 
-void kfree(void* address)
+void kfree(void* ptr)
 {
-	if (address == nullptr)
+	if (ptr == nullptr)
 		return;
 
-	uintptr_t address_uint = (uintptr_t)address;
-	ASSERT(address_uint % s_kmalloc_min_align == 0);
-
 	Kernel::SpinLockGuard _(s_kmalloc_lock);
 
-	if (s_kmalloc_fixed_info.base <= address_uint && address_uint < s_kmalloc_fixed_info.end)
-	{
-		auto& info = s_kmalloc_fixed_info;
-		ASSERT(info.used_list_head);
-
-		// get node from fixed info buffer
-		auto* node = (kmalloc_fixed_info::node*)address;
-		ASSERT(node->next < info.node_count || node->next == kmalloc_fixed_info::node::invalid);
-		ASSERT(node->prev < info.node_count || node->prev == kmalloc_fixed_info::node::invalid);
-
-		// remove from used list
-		if (node->prev != kmalloc_fixed_info::node::invalid)
-			info.node_at(node->prev)->next = node->next;
-		if (node->next != kmalloc_fixed_info::node::invalid)
-			info.node_at(node->next)->prev = node->prev;
-		if (info.used_list_head == node)
-			info.used_list_head = info.used_list_head->prev != kmalloc_fixed_info::node::invalid ? info.node_at(info.used_list_head->prev) : nullptr;
-
-		// add to free list
-		node->next = kmalloc_fixed_info::node::invalid;
-		node->prev = kmalloc_fixed_info::node::invalid;
-		if (info.free_list_head)
-		{
-			info.free_list_head->next = info.index_of(node);
-			node->prev = info.index_of(info.free_list_head);
-		}
-		info.free_list_head = node;
-
-		info.used -= sizeof(kmalloc_fixed_info::node);
-		info.free += sizeof(kmalloc_fixed_info::node);
-	}
-	else if (s_kmalloc_info.base <= address_uint && address_uint < s_kmalloc_info.end)
-	{
-		auto& info = s_kmalloc_info;
-
-		auto* node = info.from_address(address);
-		ASSERT(node);
-		ASSERT(node->data() == (uintptr_t)address);
-		ASSERT(node->used());
-
-		ptrdiff_t size = node->size_no_align();
-
-		if (auto* next = node->after(); next->end() <= info.end)
-			if (!next->used())
-				node->set_end(node->after()->end());
-		node->set_used(false);
-
-		info.used -= sizeof(kmalloc_node) + size;
-		info.free += sizeof(kmalloc_node) + size;
-	}
-	else
-	{
-		Kernel::panic("Trying to free a pointer {8H} outsize of kmalloc memory", address);
-	}
+	for (size_t i = 0; i < s_max_allocator_count && s_allocators[i]; i++)
+		if (s_allocators[i]->contains(ptr))
+			return s_allocators[i]->free(ptr);
 
-}
-
-static bool is_kmalloc_vaddr(Kernel::vaddr_t vaddr)
-{
-	using namespace Kernel;
-	if (vaddr < reinterpret_cast<vaddr_t>(s_kmalloc_storage))
-		return false;
-	if (vaddr >= reinterpret_cast<vaddr_t>(s_kmalloc_storage) + sizeof(s_kmalloc_storage))
-		return false;
-	return true;
-}
-
-BAN::Optional<Kernel::paddr_t> kmalloc_paddr_of(Kernel::vaddr_t vaddr)
-{
-	using namespace Kernel;
-	if (!is_kmalloc_vaddr(vaddr))
-		return {};
-	return vaddr - KERNEL_OFFSET + g_boot_info.kernel_paddr;
+	ASSERT_NOT_REACHED();
 }

kernel/kernel/Networking/E1000/E1000.cpp

@@ -59,7 +59,7 @@ namespace Kernel
 	E1000::~E1000()
 	{
 		m_thread_should_die = true;
-		m_thread_blocker.unblock();
+		m_rx_blocker.unblock();
 
 		while (!m_thread_is_dead)
 			Processor::yield();
@@ -214,6 +214,7 @@ namespace Kernel
 		for (size_t i = 0; i < E1000_TX_DESCRIPTOR_COUNT; i++)
 		{
 			tx_descriptors[i].addr = m_tx_buffer_region->paddr() + E1000_TX_BUFFER_SIZE * i;
+			tx_descriptors[i].status = 0xFF; /* NOTE: set status to non-zero so send_bytes doesn't think these are initially pending */
 			tx_descriptors[i].cmd = 0;
 		}
 
@@ -239,15 +240,18 @@ namespace Kernel
 	{
 		if (!link_up())
 			return 0;
-		uint32_t speed = read32(REG_STATUS) & STATUS_SPEED_MASK;
-		if (speed == STATUS_SPEED_10MB)
-			return 10;
-		if (speed == STATUS_SPEED_100MB)
-			return 100;
-		if (speed == STATUS_SPEED_1000MB1)
-			return 1000;
-		if (speed == STATUS_SPEED_1000MB2)
-			return 1000;
+
+		switch (read32(REG_STATUS) & STATUS_SPEED_MASK)
+		{
+			case STATUS_SPEED_10MB:
+				return 10;
+			case STATUS_SPEED_100MB:
+				return 100;
+			case STATUS_SPEED_1000MB1:
+			case STATUS_SPEED_1000MB2:
+				return 1000;
+		}
+
 		return 0;
 	}
 
@@ -277,9 +281,11 @@ namespace Kernel
 
 	BAN::ErrorOr<void> E1000::send_bytes(BAN::MACAddress destination, EtherType protocol, BAN::Span<const BAN::ConstByteSpan> payload)
 	{
-		SpinLockGuard _(m_lock);
+		const uint32_t tx_current = m_tx_head1.fetch_add(1) % E1000_TX_DESCRIPTOR_COUNT;
 
-		size_t tx_current = read32(REG_TDT) % E1000_TX_DESCRIPTOR_COUNT;
+		auto& descriptor = reinterpret_cast<volatile e1000_tx_desc*>(m_tx_descriptor_region->vaddr())[tx_current];
+		while (descriptor.status == 0)
+			Processor::yield();
 
 		auto* tx_buffer = reinterpret_cast<uint8_t*>(m_tx_buffer_region->vaddr() + E1000_TX_BUFFER_SIZE * tx_current);
 
@@ -296,15 +302,12 @@ namespace Kernel
 			packet_size += buffer.size();
 		}
 
-		auto& descriptor = reinterpret_cast<volatile e1000_tx_desc*>(m_tx_descriptor_region->vaddr())[tx_current];
 		descriptor.length = packet_size;
 		descriptor.status = 0;
 		descriptor.cmd = CMD_EOP | CMD_IFCS | CMD_RS;
 
-		// FIXME: there isnt really any reason to wait for transmission
-		write32(REG_TDT, (tx_current + 1) % E1000_TX_DESCRIPTOR_COUNT);
-		while (descriptor.status == 0)
-			Processor::pause();
+		if (tx_current == m_tx_head2.fetch_add(1) % E1000_TX_DESCRIPTOR_COUNT)
+			write32(REG_TDT, (tx_current + 1) % E1000_TX_DESCRIPTOR_COUNT);
 
 		dprintln_if(DEBUG_E1000, "sent {} bytes", packet_size);
 
@@ -313,7 +316,7 @@ namespace Kernel
 
 	void E1000::receive_thread()
 	{
-		SpinLockGuard _(m_lock);
+		SpinLockGuard _(m_rx_lock);
 
 		while (!m_thread_should_die)
 		{
@@ -328,21 +331,21 @@ namespace Kernel
 
 				dprintln_if(DEBUG_E1000, "got {} bytes", (uint16_t)descriptor.length);
 
-				m_lock.unlock(InterruptState::Enabled);
+				m_rx_lock.unlock(InterruptState::Enabled);
 
 				NetworkManager::get().on_receive(*this, BAN::ConstByteSpan {
 					reinterpret_cast<const uint8_t*>(m_rx_buffer_region->vaddr() + rx_current * E1000_RX_BUFFER_SIZE),
 					descriptor.length
 				});
 
-				m_lock.lock();
+				m_rx_lock.lock();
 
 				descriptor.status = 0;
 				write32(REG_RDT0, rx_current);
 			}
 
-			SpinLockAsMutex smutex(m_lock, InterruptState::Enabled);
-			m_thread_blocker.block_indefinite(&smutex);
+			SpinLockAsMutex smutex(m_rx_lock, InterruptState::Enabled);
+			m_rx_blocker.block_indefinite(&smutex);
 		}
 
 		m_thread_is_dead = true;
@@ -355,8 +358,8 @@ namespace Kernel
 
 		if (icr & (ICR_RxQ0 | ICR_RXT0))
 		{
-			SpinLockGuard _(m_lock);
-			m_thread_blocker.unblock();
+			SpinLockGuard _(m_rx_lock);
+			m_rx_blocker.unblock();
 		}
 	}
 

kernel/kernel/Networking/TCPSocket.cpp

@@ -66,6 +66,8 @@ namespace Kernel
 
 	BAN::ErrorOr<long> TCPSocket::accept_impl(sockaddr* address, socklen_t* address_len, int flags)
 	{
+		LockGuard _(m_mutex);
+
 		if (m_state != State::Listen)
 			return BAN::Error::from_errno(EINVAL);
 
@@ -171,6 +173,8 @@ namespace Kernel
 
 	BAN::ErrorOr<void> TCPSocket::listen_impl(int backlog)
 	{
+		LockGuard _(m_mutex);
+
 		if (!is_bound())
 			return BAN::Error::from_errno(EDESTADDRREQ);
 		if (m_connection_info.has_value())
@@ -185,6 +189,8 @@ namespace Kernel
 
 	BAN::ErrorOr<void> TCPSocket::bind_impl(const sockaddr* address, socklen_t address_len)
 	{
+		LockGuard _(m_mutex);
+
 		if (is_bound())
 			return BAN::Error::from_errno(EINVAL);
 		return m_network_layer.bind_socket_to_address(this, address, address_len);
@@ -204,6 +210,8 @@ namespace Kernel
 			message.msg_controllen = 0;
 		}
 
+		LockGuard _(m_mutex);
+
 		if (!m_has_connected)
 			return BAN::Error::from_errno(ENOTCONN);
 
@@ -261,6 +269,8 @@ namespace Kernel
 		if (CMSG_FIRSTHDR(&message))
 			dwarnln("ignoring sendmsg control message");
 
+		LockGuard _(m_mutex);
+
 		if (!m_has_connected)
 			return BAN::Error::from_errno(ENOTCONN);
 
@@ -291,6 +301,7 @@ namespace Kernel
 
 	BAN::ErrorOr<void> TCPSocket::getpeername_impl(sockaddr* address, socklen_t* address_len)
 	{
+		LockGuard _(m_mutex);
 		if (!m_has_connected && m_state != State::Established)
 			return BAN::Error::from_errno(ENOTCONN);
 		ASSERT(m_connection_info.has_value());
@@ -302,6 +313,8 @@ namespace Kernel
 
 	BAN::ErrorOr<void> TCPSocket::getsockopt_impl(int level, int option, void* value, socklen_t* value_len)
 	{
+		LockGuard _(m_mutex);
+
 		int result;
 
 		switch (level)
@@ -351,6 +364,8 @@ namespace Kernel
 
 	BAN::ErrorOr<void> TCPSocket::setsockopt_impl(int level, int option, const void* value, socklen_t value_len)
 	{
+		LockGuard _(m_mutex);
+
 		switch (level)
 		{
 			case SOL_SOCKET:
@@ -401,6 +416,7 @@ namespace Kernel
 
 	bool TCPSocket::can_read_impl() const
 	{
+		LockGuard _(m_mutex);
 		if (m_has_connected && !m_has_sent_zero && m_state != State::Established && m_state != State::Listen)
 			return true;
 		if (m_state == State::Listen)
@@ -410,6 +426,7 @@ namespace Kernel
 
 	bool TCPSocket::can_write_impl() const
 	{
+		LockGuard _(m_mutex);
 		if (m_state != State::Established)
 			return false;
 		return !m_send_window.buffer->full();
@@ -417,6 +434,7 @@ namespace Kernel
 
 	bool TCPSocket::has_hungup_impl() const
 	{
+		LockGuard _(m_mutex);
 		return m_has_connected && m_state != State::Established;
 	}
 

kernel/kernel/Networking/UDPSocket.cpp

@@ -1,3 +1,4 @@
+#include <kernel/Lock/LockGuard.h>
 #include <kernel/Lock/SpinLockAsMutex.h>
 #include <kernel/Memory/Heap.h>
 #include <kernel/Networking/UDPSocket.h>
@@ -58,9 +59,6 @@ namespace Kernel
 
 	void UDPSocket::receive_packet(BAN::ConstByteSpan packet, const sockaddr* sender, socklen_t sender_len)
 	{
-		(void)sender_len;
-
-		//auto& header = packet.as<const UDPHeader>();
 		auto payload = packet.slice(sizeof(UDPHeader));
 
 		SpinLockGuard _(m_packet_lock);
@@ -95,6 +93,8 @@ namespace Kernel
 	{
 		if (address_len > static_cast<socklen_t>(sizeof(m_peer_address)))
 			address_len = sizeof(m_peer_address);
+
+		SpinLockGuard _(m_peer_address_lock);
 		memcpy(&m_peer_address, address, address_len);
 		m_peer_address_len = address_len;
 		return {};
@@ -102,6 +102,7 @@ namespace Kernel
 
 	BAN::ErrorOr<void> UDPSocket::bind_impl(const sockaddr* address, socklen_t address_len)
 	{
+		LockGuard _(m_bind_lock);
 		if (is_bound())
 			return BAN::Error::from_errno(EINVAL);
 		return m_network_layer.bind_socket_to_address(this, address, address_len);
@@ -183,8 +184,11 @@ namespace Kernel
 		if (CMSG_FIRSTHDR(&message))
 			dwarnln("ignoring sendmsg control message");
 
-		if (!is_bound())
-			TRY(m_network_layer.bind_socket_with_target(this, static_cast<sockaddr*>(message.msg_name), message.msg_namelen));
+		{
+			LockGuard _(m_bind_lock);
+			if (!is_bound())
+				TRY(m_network_layer.bind_socket_with_target(this, static_cast<sockaddr*>(message.msg_name), message.msg_namelen));
+		}
 
 		const size_t total_send_size =
 			[&message]() -> size_t {
@@ -208,6 +212,7 @@ namespace Kernel
 		socklen_t address_len;
 		if (!message.msg_name || message.msg_namelen == 0)
 		{
+			SpinLockGuard _(m_peer_address_lock);
 			if (m_peer_address_len == 0)
 				return BAN::Error::from_errno(EDESTADDRREQ);
 			address = reinterpret_cast<sockaddr*>(&m_peer_address);

kernel/kernel/Networking/UNIX/Socket.cpp

@@ -88,6 +88,26 @@ namespace Kernel
 		}
 	}
 
+	bool UnixDomainSocket::is_bound() const
+	{
+		LockGuard _(m_bind_mutex);
+		return !m_bound_file.canonical_path.empty();
+	}
+
+	bool UnixDomainSocket::is_bound_to_unused() const
+	{
+		LockGuard _(m_bind_mutex);
+		return !m_bound_file.inode;
+	}
+
+	BAN::ErrorOr<void> UnixDomainSocket::bind_to_unused_if_not_bound()
+	{
+		LockGuard _(m_bind_mutex);
+		if (!is_bound())
+			TRY(m_bound_file.canonical_path.push_back('X'));
+		return {};
+	}
+
 	BAN::ErrorOr<void> UnixDomainSocket::make_socket_pair(UnixDomainSocket& other)
 	{
 		if (!m_info.has<ConnectionInfo>() || !other.m_info.has<ConnectionInfo>())
@@ -106,15 +126,15 @@ namespace Kernel
 	{
 		if (!m_info.has<ConnectionInfo>())
 			return BAN::Error::from_errno(EOPNOTSUPP);
-		auto& connection_info = m_info.get<ConnectionInfo>();
-		if (!connection_info.listening)
-			return BAN::Error::from_errno(EINVAL);
-
 
 		BAN::RefPtr<UnixDomainSocket> pending;
 
 		{
+			auto& connection_info = m_info.get<ConnectionInfo>();
+
 			LockGuard _(connection_info.pending_lock);
+			if (!connection_info.listening)
+				return BAN::Error::from_errno(EINVAL);
 			while (connection_info.pending_connections.empty())
 				TRY(Thread::current().block_or_eintr_indefinite(connection_info.pending_thread_blocker, &connection_info.pending_lock));
 
@@ -154,8 +174,6 @@ namespace Kernel
 	BAN::ErrorOr<void> UnixDomainSocket::connect_impl(const sockaddr* address, socklen_t address_len)
 	{
 		const auto sun_path = TRY(validate_sockaddr_un(address, address_len));
-		if (!is_bound())
-			TRY(m_bound_file.canonical_path.push_back('X'));
 
 		auto absolute_path = TRY(Process::current().absolute_path_of(sun_path));
 		auto file = TRY(VirtualFileSystem::get().file_from_absolute_path(
@@ -180,9 +198,12 @@ namespace Kernel
 		if (m_socket_type != target->m_socket_type)
 			return BAN::Error::from_errno(EPROTOTYPE);
 
+		TRY(bind_to_unused_if_not_bound());
+
 		if (m_info.has<ConnectionlessInfo>())
 		{
 			auto& connectionless_info = m_info.get<ConnectionlessInfo>();
+			SpinLockGuard _(connectionless_info.lock);
 			connectionless_info.peer_address = BAN::move(file.canonical_path);
 			return {};
 		}
@@ -192,7 +213,6 @@ namespace Kernel
 			return BAN::Error::from_errno(ECONNREFUSED);
 		if (connection_info.listening)
 			return BAN::Error::from_errno(EOPNOTSUPP);
-
 		connection_info.connection_done = false;
 
 		for (;;)
@@ -222,23 +242,25 @@ namespace Kernel
 	BAN::ErrorOr<void> UnixDomainSocket::listen_impl(int backlog)
 	{
 		backlog = BAN::Math::clamp(backlog, 1, SOMAXCONN);
+
 		if (!is_bound())
 			return BAN::Error::from_errno(EDESTADDRREQ);
 		if (!m_info.has<ConnectionInfo>())
 			return BAN::Error::from_errno(EOPNOTSUPP);
+
 		auto& connection_info = m_info.get<ConnectionInfo>();
+
+		LockGuard _(connection_info.pending_lock);
 		if (connection_info.connection)
 			return BAN::Error::from_errno(EINVAL);
 		TRY(connection_info.pending_connections.reserve(backlog));
 		connection_info.listening = true;
+
 		return {};
 	}
 
 	BAN::ErrorOr<void> UnixDomainSocket::bind_impl(const sockaddr* address, socklen_t address_len)
 	{
-		if (is_bound())
-			return BAN::Error::from_errno(EINVAL);
-
 		const auto sun_path = TRY(validate_sockaddr_un(address, address_len));
 		if (sun_path.empty())
 			return BAN::Error::from_errno(EINVAL);
@@ -261,10 +283,15 @@ namespace Kernel
 			O_RDWR
 		));
 
-		LockGuard _(s_bound_socket_lock);
+		LockGuard _0(m_bind_mutex);
+		if (is_bound())
+			return BAN::Error::from_errno(EINVAL);
+
+		LockGuard _1(s_bound_socket_lock);
 		if (s_bound_sockets.contains(file.inode))
 			return BAN::Error::from_errno(EADDRINUSE);
 		TRY(s_bound_sockets.emplace(file.inode, TRY(get_weak_ptr())));
+
 		m_bound_file = BAN::move(file);
 
 		return {};
@@ -679,6 +706,7 @@ namespace Kernel
 	{
 		if (!m_info.has<ConnectionInfo>())
 			return BAN::Error::from_errno(ENOTCONN);
+
 		auto connection = m_info.get<ConnectionInfo>().connection.lock();
 		if (!connection)
 			return BAN::Error::from_errno(ENOTCONN);
@@ -687,7 +715,11 @@ namespace Kernel
 			.sun_family = AF_UNIX,
 			.sun_path = {},
 		};
-		strcpy(sa_un.sun_path, connection->m_bound_file.canonical_path.data());
+
+		{
+			LockGuard _(m_bind_mutex);
+			strcpy(sa_un.sun_path, connection->m_bound_file.canonical_path.data());
+		}
 
 		const size_t to_copy = BAN::Math::min<socklen_t>(sizeof(sockaddr_un), *address_len);
 		memcpy(address, &sa_un, to_copy);

kernel/kernel/OpenFileDescriptorSet.cpp

@@ -699,9 +699,9 @@ namespace Kernel
 
 		size_t nread;
 		{
-			LockGuard _(inode->m_mutex);
 			if (!inode->can_read() && inode->has_hungup())
 				return 0;
+			// FIXME: race condition, pass flags to read
 			if (is_nonblock && !inode->can_read())
 				return BAN::Error::from_errno(EAGAIN);
 			nread = TRY(inode->read(offset, buffer));
@@ -753,7 +753,6 @@ namespace Kernel
 
 		size_t nwrite;
 		{
-			LockGuard _(inode->m_mutex);
 			if (inode->has_error())
 			{
 				Thread::current().add_signal(SIGPIPE, {});
@@ -761,6 +760,7 @@ namespace Kernel
 			}
 			if (is_nonblock && !inode->can_write())
 				return BAN::Error::from_errno(EAGAIN);
+			// FIXME: race condition, pass flags to write
 			nwrite = TRY(inode->write(offset, buffer));
 		}
 
@@ -818,7 +818,6 @@ namespace Kernel
 			is_nonblock = !!(open_file->status_flags & O_NONBLOCK);
 		}
 
-		LockGuard _(inode->m_mutex);
 		if (is_nonblock && !inode->can_read())
 			return BAN::Error::from_errno(EAGAIN);
 		return inode->recvmsg(message, flags);
@@ -839,7 +838,6 @@ namespace Kernel
 			is_nonblock = !!(open_file->status_flags & O_NONBLOCK);
 		}
 
-		LockGuard _(inode->m_mutex);
 		if (inode->has_hungup())
 		{
 			if (!(flags & MSG_NOSIGNAL))

kernel/kernel/PCI.cpp

@@ -45,34 +45,58 @@ namespace Kernel::PCI
 	};
 	static_assert(sizeof(MSIXEntry) == 16);
 
+	static uint32_t get_device_io_address(uint8_t bus, uint8_t dev, uint8_t func)
+	{
+		 return 0x80000000
+			| (static_cast<uint32_t>(bus) << 16)
+			| (static_cast<uint32_t>(dev) << 11)
+			| (static_cast<uint32_t>(func) << 8);
+	}
+
 	uint32_t PCIManager::read_config_dword(uint8_t bus, uint8_t dev, uint8_t func, uint8_t offset)
 	{
-		return m_buses[bus][dev][func].read_dword(offset);
+		ASSERT(offset % 4 == 0);
+		IO::outl(CONFIG_ADDRESS, get_device_io_address(bus, dev, func) | offset);
+		return IO::inl(CONFIG_DATA);
 	}
 
 	uint16_t PCIManager::read_config_word(uint8_t bus, uint8_t dev, uint8_t func, uint8_t offset)
 	{
-		return m_buses[bus][dev][func].read_word(offset);
+		ASSERT(offset % 2 == 0);
+		const uint32_t dword = read_config_dword(bus, dev, func, offset & ~3);
+		return (dword >> ((offset & 3) * 8)) & 0xFFFF;
 	}
 
 	uint8_t PCIManager::read_config_byte(uint8_t bus, uint8_t dev, uint8_t func, uint8_t offset)
 	{
-		return m_buses[bus][dev][func].read_byte(offset);
+		const uint32_t dword = read_config_dword(bus, dev, func, offset & ~3);
+		return (dword >> ((offset & 3) * 8)) & 0xFF;
 	}
 
 	void PCIManager::write_config_dword(uint8_t bus, uint8_t dev, uint8_t func, uint8_t offset, uint32_t value)
 	{
-		m_buses[bus][dev][func].write_dword(offset, value);
+		ASSERT(offset % 4 == 0);
+		IO::outl(CONFIG_ADDRESS, get_device_io_address(bus, dev, func) | offset);
+		IO::outl(CONFIG_DATA, value);
 	}
 
 	void PCIManager::write_config_word(uint8_t bus, uint8_t dev, uint8_t func, uint8_t offset, uint16_t value)
 	{
-		m_buses[bus][dev][func].write_word(offset, value);
+		ASSERT(offset % 2 == 0);
+		const uint32_t byte = (offset & 3) * 8;
+		uint32_t temp = read_config_dword(bus, dev, func, offset & ~3);
+		temp &= ~(0xFFFF << byte);
+		temp |= (uint32_t)value << byte;
+		write_config_dword(bus, dev, func, offset & ~3, temp);
 	}
 
 	void PCIManager::write_config_byte(uint8_t bus, uint8_t dev, uint8_t func, uint8_t offset, uint8_t value)
 	{
-		m_buses[bus][dev][func].write_byte(offset, value);
+		const uint32_t byte = (offset & 3) * 8;
+		uint32_t temp = read_config_dword(bus, dev, func, offset & ~3);
+		temp &= ~(0xFF << byte);
+		temp |= (uint32_t)value << byte;
+		write_config_dword(bus, dev, func, offset & ~3, temp);
 	}
 
 	static uint16_t get_vendor_id(uint8_t bus, uint8_t dev, uint8_t func)
@@ -107,6 +131,9 @@ namespace Kernel::PCI
 		};
 		static_assert(sizeof(BAAS) == 16);
 
+		auto* pcie_info = new PCIeInfo;
+		ASSERT(pcie_info);
+
 		if (auto* mcfg = ACPI::ACPI::get().get_header("MCFG", 0))
 		{
 			const size_t count = (mcfg->length - 44) / 16;
@@ -118,19 +145,15 @@ namespace Kernel::PCI
 					continue;
 				for (uint64_t bus = baas->bus_start; bus <= baas->bus_end; bus++)
 				{
-					ASSERT(m_bus_pcie_paddr[bus] == 0);
-					m_bus_pcie_paddr[bus] = baas->addr + (bus << 20);
+					ASSERT(pcie_info->bus_paddr[bus] == 0);
+					pcie_info->bus_paddr[bus] = baas->addr + (bus << 20);
 				}
 			}
-
-			m_is_pcie = true;
 		}
 
-		for (size_t bus = 0; bus < m_buses.size(); bus++)
-			for (size_t dev = 0; dev < m_buses[bus].size(); dev++)
-				for (size_t func = 0; func < m_buses[bus][dev].size(); func++)
-					m_buses[bus][dev][func].set_location(bus, dev, func);
-		s_instance->check_all_buses();
+		check_all_buses(*pcie_info);
+
+		delete pcie_info;
 	}
 
 	PCIManager& PCIManager::get()
@@ -139,43 +162,47 @@ namespace Kernel::PCI
 		return *s_instance;
 	}
 
-	void PCIManager::check_function(uint8_t bus, uint8_t dev, uint8_t func)
+	void PCIManager::check_function(const PCIeInfo& pcie_info, uint8_t bus, uint8_t dev, uint8_t func)
 	{
-		auto& device = m_buses[bus][dev][func];
-		const paddr_t pcie_paddr = m_is_pcie ? m_bus_pcie_paddr[bus] + (((paddr_t)dev << 15) | ((paddr_t)func << 12)) : 0;
+		const paddr_t pcie_paddr = pcie_info.bus_paddr[bus] ? pcie_info.bus_paddr[bus] + (((paddr_t)dev << 15) | ((paddr_t)func << 12)) : 0;
+
+		MUST(m_devices.emplace_back(bus, dev, func));
+
+		auto& device = m_devices.back();
 		device.initialize(pcie_paddr);
+
 		if (device.class_code() == 0x06 && device.subclass() == 0x04)
-			check_bus(device.read_byte(0x19));
+			check_bus(pcie_info, device.read_byte(0x19));
 	}
 
-	void PCIManager::check_device(uint8_t bus, uint8_t dev)
+	void PCIManager::check_device(const PCIeInfo& pcie_info, uint8_t bus, uint8_t dev)
 	{
 		if (get_vendor_id(bus, dev, 0) == INVALID_VENDOR)
 			return;
 
-		check_function(bus, dev, 0);
+		check_function(pcie_info, bus, dev, 0);
 		if (get_header_type(bus, dev, 0) & MULTI_FUNCTION)
 			for (uint8_t func = 1; func < 8; func++)
 				if (get_vendor_id(bus, dev, func) != INVALID_VENDOR)
-					check_function(bus, dev, func);
+					check_function(pcie_info, bus, dev, func);
 	}
 
-	void PCIManager::check_bus(uint8_t bus)
+	void PCIManager::check_bus(const PCIeInfo& pcie_info, uint8_t bus)
 	{
 		for (uint8_t dev = 0; dev < 32; dev++)
-			check_device(bus, dev);
+			check_device(pcie_info, bus, dev);
 	}
 
-	void PCIManager::check_all_buses()
+	void PCIManager::check_all_buses(const PCIeInfo& pcie_info)
 	{
 		if (get_header_type(0, 0, 0) & MULTI_FUNCTION)
 		{
 			for (int func = 0; func < 8 && get_vendor_id(0, 0, func) != INVALID_VENDOR; func++)
-				check_bus(func);
+				check_bus(pcie_info, func);
 		}
 		else
 		{
-			check_bus(0);
+			check_bus(pcie_info, 0);
 		}
 	}
 
@@ -264,32 +291,23 @@ namespace Kernel::PCI
 		);
 	}
 
-	void PCI::Device::set_location(uint8_t bus, uint8_t dev, uint8_t func)
-	{
-		m_bus = bus;
-		m_dev = dev;
-		m_func = func;
-	}
-
 	void PCI::Device::initialize(paddr_t pcie_paddr)
 	{
-		m_is_valid = true;
-
 		if (pcie_paddr)
 		{
-			vaddr_t vaddr = PageTable::kernel().reserve_free_page(KERNEL_OFFSET);
+			const vaddr_t vaddr = PageTable::kernel().reserve_free_page(KERNEL_OFFSET);
 			ASSERT(vaddr);
 			PageTable::kernel().map_page_at(pcie_paddr, vaddr, PageTable::Flags::ReadWrite | PageTable::Flags::Present, PageTable::MemoryType::Uncached);
 			m_mmio_config = vaddr;
 		}
 
-		uint32_t type = read_word(0x0A);
+		const uint32_t type = read_word(0x0A);
 		m_class_code  = (uint8_t)(type >> 8);
 		m_subclass    = (uint8_t)(type);
 		m_prog_if     = read_byte(0x09);
 		m_header_type = read_byte(0x0E);
 
-		uint32_t device = read_dword(0x00);
+		const uint32_t device = read_dword(0x00);
 		m_vendor_id = device & 0xFFFF;
 		m_device_id = device >> 16;
 
@@ -303,62 +321,44 @@ namespace Kernel::PCI
 
 	uint32_t PCI::Device::read_dword(uint8_t offset) const
 	{
-		ASSERT(offset % 4 == 0);
 		if (m_mmio_config)
 			return MMIO::read32(m_mmio_config + offset);
-		uint32_t config_addr = 0x80000000 | ((uint32_t)m_bus << 16) | ((uint32_t)m_dev << 11) | ((uint32_t)m_func << 8) | offset;
-		IO::outl(CONFIG_ADDRESS, config_addr);
-		return IO::inl(CONFIG_DATA);
+		return s_instance->read_config_dword(m_bus, m_dev, m_func, offset);
 	}
 
 	uint16_t PCI::Device::read_word(uint8_t offset) const
 	{
-		ASSERT(offset % 2 == 0);
 		if (m_mmio_config)
 			return MMIO::read16(m_mmio_config + offset);
-		uint32_t dword = read_dword(offset & ~3);
-		return (dword >> ((offset & 3) * 8)) & 0xFFFF;
+		return s_instance->read_config_word(m_bus, m_dev, m_func, offset);
 	}
 
 	uint8_t PCI::Device::read_byte(uint8_t offset) const
 	{
 		if (m_mmio_config)
 			return MMIO::read8(m_mmio_config + offset);
-		uint32_t dword = read_dword(offset & ~3);
-		return (dword >> ((offset & 3) * 8)) & 0xFF;
+		return s_instance->read_config_byte(m_bus, m_dev, m_func, offset);
 	}
 
 	void PCI::Device::write_dword(uint8_t offset, uint32_t value)
 	{
-		ASSERT(offset % 4 == 0);
 		if (m_mmio_config)
 			return MMIO::write32(m_mmio_config + offset, value);
-		uint32_t config_addr = 0x80000000 | ((uint32_t)m_bus << 16) | ((uint32_t)m_dev << 11) | ((uint32_t)m_func << 8) | offset;
-		IO::outl(CONFIG_ADDRESS, config_addr);
-		IO::outl(CONFIG_DATA, value);
+		s_instance->write_config_dword(m_bus, m_dev, m_func, offset, value);
 	}
 
 	void PCI::Device::write_word(uint8_t offset, uint16_t value)
 	{
-		ASSERT(offset % 2 == 0);
 		if (m_mmio_config)
 			return MMIO::write16(m_mmio_config + offset, value);
-		uint32_t byte = (offset & 3) * 8;
-		uint32_t temp = read_dword(offset & ~3);
-		temp &= ~(0xFFFF << byte);
-		temp |= (uint32_t)value << byte;
-		write_dword(offset & ~3, temp);
+		s_instance->write_config_word(m_bus, m_dev, m_func, offset, value);
 	}
 
 	void PCI::Device::write_byte(uint8_t offset, uint8_t value)
 	{
 		if (m_mmio_config)
 			return MMIO::write8(m_mmio_config + offset, value);
-		uint32_t byte = (offset & 3) * 8;
-		uint32_t temp = read_dword(offset & ~3);
-		temp &= ~(0xFF << byte);
-		temp |= (uint32_t)value << byte;
-		write_dword(offset & ~3, temp);
+		s_instance->write_config_byte(m_bus, m_dev, m_func, offset, value);
 	}
 
 	BAN::ErrorOr<BAN::UniqPtr<BarRegion>> PCI::Device::allocate_bar_region(uint8_t bar_num)

kernel/kernel/Processor.cpp

@@ -464,7 +464,7 @@ namespace Kernel
 		auto* storage = processor.m_smp_free.exchange(nullptr);
 		while (storage == nullptr)
 		{
-			__builtin_ia32_pause();
+			Processor::pause();
 			storage = processor.m_smp_free.exchange(nullptr);
 		}
 

kernel/kernel/Random.cpp

@@ -1,5 +1,6 @@
 #include <kernel/CPUID.h>
 #include <kernel/Debug.h>
+#include <kernel/Lock/SpinLock.h>
 #include <kernel/Random.h>
 #include <kernel/Timer/Timer.h>
 
@@ -9,6 +10,7 @@ namespace Kernel
 
 	// Constants and algorithm from https://en.wikipedia.org/wiki/Permuted_congruential_generator
 
+	static SpinLock s_rand_lock;
 	static uint64_t s_rand_seed = 0x4d595df4d0f33173;
 	static constexpr uint64_t s_rand_multiplier = 6364136223846793005;
 	static constexpr uint64_t s_rand_increment = 1442695040888963407;
@@ -46,7 +48,9 @@ namespace Kernel
 
 	uint32_t Random::get_u32()
 	{
-		auto rotr32 = [](uint32_t x, unsigned r) { return x >> r | x << (-r & 31); };
+		constexpr auto rotr32 = [](uint32_t x, unsigned r) { return x >> r | x << (-r & 31); };
+
+		SpinLockGuard _(s_rand_lock);
 
 		uint64_t x = s_rand_seed;
 		unsigned count = (unsigned)(x >> 59);

kernel/kernel/Terminal/PseudoTerminal.cpp

@@ -144,6 +144,7 @@ namespace Kernel
 		auto slave = m_slave.lock();
 		if (!slave)
 			return BAN::Error::from_errno(EIO);
+		LockGuard _(slave->m_mutex);
 		for (size_t i = 0; i < buffer.size(); i++)
 			slave->handle_input_byte(buffer[i]);
 		return buffer.size();
@@ -160,8 +161,11 @@ namespace Kernel
 		switch (request)
 		{
 			case FIONREAD:
+			{
+				SpinLockGuard _(m_buffer_lock);
 				*static_cast<int*>(argument) = m_buffer_size;
 				return 0;
+			}
 			case TIOCGWINSZ:
 			case TIOCSWINSZ:
 				if (auto slave = m_slave.lock())

kernel/kernel/Terminal/TTY.cpp

@@ -6,6 +6,7 @@
 #include <kernel/FS/DevFS/FileSystem.h>
 #include <kernel/FS/VirtualFileSystem.h>
 #include <kernel/Lock/LockGuard.h>
+#include <kernel/Lock/SpinLockAsMutex.h>
 #include <kernel/Process.h>
 #include <kernel/Terminal/TTY.h>
 #include <kernel/Timer/Timer.h>
@@ -71,8 +72,8 @@ namespace Kernel
 
 	TTY::TTY(termios termios, mode_t mode, uid_t uid, gid_t gid)
 		: CharacterDevice(mode, uid, gid)
-		, m_termios(termios)
 		, m_rdev(next_tty_rdev())
+		, m_termios(termios)
 	{
 		m_output.buffer = MUST(ByteRingBuffer::create(PAGE_SIZE));
 	}
@@ -94,21 +95,16 @@ namespace Kernel
 		if (flags & ~(TTY_FLAG_ENABLE_INPUT | TTY_FLAG_ENABLE_OUTPUT))
 			return BAN::Error::from_errno(EINVAL);
 
-		LockGuard _(m_mutex);
-
 		switch (command)
 		{
 			case TTY_CMD_SET:
-				if ((flags & TTY_FLAG_ENABLE_INPUT) && !m_tty_ctrl.receive_input)
-				{
+				if (flags & TTY_FLAG_ENABLE_INPUT)
 					m_tty_ctrl.receive_input = true;
-					m_tty_ctrl.thread_blocker.unblock();
-				}
 				if (flags & TTY_FLAG_ENABLE_OUTPUT)
 					m_tty_ctrl.draw_graphics = true;
 				break;
 			case TTY_CMD_UNSET:
-				if ((flags & TTY_FLAG_ENABLE_INPUT) && m_tty_ctrl.receive_input)
+				if (flags & TTY_FLAG_ENABLE_INPUT)
 					m_tty_ctrl.receive_input = false;
 				if (flags & TTY_FLAG_ENABLE_OUTPUT)
 					m_tty_ctrl.draw_graphics = false;
@@ -133,16 +129,8 @@ namespace Kernel
 
 		while (true)
 		{
-			{
-				LockGuard _(TTY::current()->m_mutex);
-				while (!TTY::current()->m_tty_ctrl.receive_input)
-					TTY::current()->m_tty_ctrl.thread_blocker.block_indefinite(&TTY::current()->m_mutex);
-			}
-
 			while (TTY::current()->m_tty_ctrl.receive_input)
 			{
-				LockGuard _(keyboard_inode->m_mutex);
-
 				if (!keyboard_inode->can_read())
 				{
 					SystemTimer::get().sleep_ms(1);
@@ -160,19 +148,15 @@ namespace Kernel
 
 	void TTY::initialize_devices()
 	{
-		static bool initialized = false;
-		ASSERT(!initialized);
-
 		auto* thread = MUST(Thread::create_kernel(&TTY::keyboard_task, nullptr));
 		MUST(Processor::scheduler().add_thread(thread));
 
 		DevFileSystem::get().add_inode("tty", MUST(DevTTY::create(0666, 0, 0)));
-
-		initialized = true;
 	}
 
 	BAN::ErrorOr<void> TTY::chmod_impl(mode_t mode)
 	{
+		// FIXME: make this atomic
 		ASSERT((mode & Inode::Mode::TYPE_MASK) == 0);
 		m_inode_info.mode &= Inode::Mode::TYPE_MASK;
 		m_inode_info.mode |= mode;
@@ -181,6 +165,7 @@ namespace Kernel
 
 	BAN::ErrorOr<void> TTY::chown_impl(uid_t uid, gid_t gid)
 	{
+		// FIXME: make this atomic
 		m_inode_info.uid = uid;
 		m_inode_info.gid = gid;
 		return {};
@@ -188,6 +173,7 @@ namespace Kernel
 
 	void TTY::update_winsize(unsigned short cols, unsigned short rows)
 	{
+		// FIXME: make this atomic
 		m_winsize.ws_col = cols;
 		m_winsize.ws_row = rows;
 		(void)Process::kill(-m_foreground_pgrp, SIGWINCH);
@@ -211,12 +197,14 @@ namespace Kernel
 			}
 			case TIOCGWINSZ:
 			{
+				// FIXME: make this atomic
 				auto* winsize = static_cast<struct winsize*>(argument);
 				*winsize = m_winsize;
 				return 0;
 			}
 			case TIOCSWINSZ:
 			{
+				// FIXME: make this atomic
 				const auto* winsize = static_cast<const struct winsize*>(argument);
 				m_winsize = *winsize;
 				(void)Process::kill(-m_foreground_pgrp, SIGWINCH);
@@ -226,10 +214,13 @@ namespace Kernel
 		return BAN::Error::from_errno(ENOTSUP);
 	}
 
+	void TTY::on_key_event(LibInput::RawKeyEvent event)
+	{
+		on_key_event(LibInput::KeyboardLayout::get().key_event_from_raw(event));
+	}
+
 	void TTY::on_key_event(LibInput::KeyEvent event)
 	{
-		LockGuard _(m_mutex);
-
 		if (event.released())
 			return;
 
@@ -237,35 +228,55 @@ namespace Kernel
 		if (ansi_c_str == nullptr)
 			return;
 
+		LockGuard _(m_mutex);
 		while (*ansi_c_str)
 			handle_input_byte(*ansi_c_str++);
 		after_write();
 	}
 
+	void TTY::get_termios(termios* termios)
+	{
+		SpinLockGuard _(m_termios_lock);
+		*termios = m_termios;
+	}
+
+	BAN::ErrorOr<void> TTY::set_termios(const termios* termios)
+	{
+		// FIXME: do some validation
+
+		SpinLockGuard _(m_termios_lock);
+		m_termios = *termios;
+		return {};
+	}
+
 	void TTY::handle_input_byte(uint8_t ch)
 	{
 		if (ch == _POSIX_VDISABLE)
 			return;
 
-		LockGuard _(m_mutex);
+		LockGuard _0(m_mutex);
 
-		if ((m_termios.c_iflag & ISTRIP))
+		termios termios;
+		get_termios(&termios);
+
+
+		if ((termios.c_iflag & ISTRIP))
 			ch &= 0x7F;
-		if ((m_termios.c_iflag & IGNCR) && ch == CR)
+		if ((termios.c_iflag & IGNCR) && ch == CR)
 			return;
-		if ((m_termios.c_iflag & ICRNL) && ch == CR)
+		if ((termios.c_iflag & ICRNL) && ch == CR)
 			ch = NL;
-		else if ((m_termios.c_iflag & INLCR) && ch == NL)
+		else if ((termios.c_iflag & INLCR) && ch == NL)
 			ch = CR;
 
-		if (m_termios.c_lflag & ISIG)
+		if (termios.c_lflag & ISIG)
 		{
 			int sig = -1;
-			if (ch == m_termios.c_cc[VINTR])
+			if (ch == termios.c_cc[VINTR])
 				sig = SIGINT;
-			if (ch == m_termios.c_cc[VQUIT])
+			if (ch == termios.c_cc[VQUIT])
 				sig = SIGQUIT;
-			if (ch == m_termios.c_cc[VSUSP])
+			if (ch == termios.c_cc[VSUSP])
 				sig = SIGTSTP;
 			if (sig != -1)
 			{
@@ -279,30 +290,30 @@ namespace Kernel
 		bool should_flush = false;
 		bool force_echo = false;
 
-		if (!(m_termios.c_lflag & ICANON))
+		if (!(termios.c_lflag & ICANON))
 			should_flush = true;
 		else
 		{
-			if (ch == m_termios.c_cc[VERASE] && (m_termios.c_lflag & ECHOE))
+			if (ch == termios.c_cc[VERASE] && (termios.c_lflag & ECHOE))
 				return do_backspace();
 
-			if (ch == m_termios.c_cc[VKILL] && (m_termios.c_lflag & ECHOK))
+			if (ch == termios.c_cc[VKILL] && (termios.c_lflag & ECHOK))
 			{
 				while (!m_output.buffer->empty() && m_output.buffer->back() != '\n')
 					do_backspace();
 				return;
 			}
 
-			if (ch == m_termios.c_cc[VEOF])
+			if (ch == termios.c_cc[VEOF])
 			{
 				should_append = false;
 				should_flush = true;
 			}
 
-			if (ch == NL || ch == CR || ch == m_termios.c_cc[VEOL])
+			if (ch == NL || ch == CR || ch == termios.c_cc[VEOL])
 			{
 				should_flush = true;
-				force_echo = !!(m_termios.c_lflag & ECHONL);
+				force_echo = !!(termios.c_lflag & ECHONL);
 			}
 		}
 
@@ -311,7 +322,7 @@ namespace Kernel
 		if (should_append && !m_output.buffer->full())
 			m_output.buffer->push({ &ch, 1 });
 
-		if (should_append && (force_echo || (m_termios.c_lflag & ECHO)))
+		if (should_append && (force_echo || (termios.c_lflag & ECHO)))
 		{
 			if ((ch <= 31 || ch == 127) && ch != '\n')
 			{
@@ -380,14 +391,18 @@ namespace Kernel
 
 	bool TTY::putchar(uint8_t ch)
 	{
-		SpinLockGuard _(m_write_lock);
 		if (!m_tty_ctrl.draw_graphics)
 			return true;
-		if (m_termios.c_oflag & OPOST)
+
+		termios termios;
+		get_termios(&termios);
+
+		SpinLockGuard _1(m_write_lock);
+		if (termios.c_oflag & OPOST)
 		{
-			if ((m_termios.c_oflag & ONLCR) && ch == NL)
+			if ((termios.c_oflag & ONLCR) && ch == NL)
 				return putchar_impl(CR) && putchar_impl(NL);
-			if ((m_termios.c_oflag & OCRNL) && ch == CR)
+			if ((termios.c_oflag & OCRNL) && ch == CR)
 				return putchar_impl(NL);
 		}
 		return putchar_impl(ch);
@@ -395,6 +410,7 @@ namespace Kernel
 
 	BAN::ErrorOr<size_t> TTY::read_impl(off_t, BAN::ByteSpan buffer)
 	{
+		LockGuard _(m_mutex);
 		while (!m_output.flush)
 			TRY(Thread::current().block_or_eintr_indefinite(m_output.thread_blocker, &m_mutex));
 
@@ -428,24 +444,24 @@ namespace Kernel
 
 	BAN::ErrorOr<size_t> TTY::write_impl(off_t, BAN::ConstByteSpan buffer)
 	{
-		while (!can_write_impl())
+		SpinLockGuard write_guard(m_write_lock);
+
+		while (!can_write())
 		{
 			if (master_has_closed())
 				return BAN::Error::from_errno(EIO);
-			TRY(Thread::current().block_or_eintr_indefinite(m_write_blocker, &m_mutex));
+
+			SpinLockGuardAsMutex smutex(write_guard);
+			TRY(Thread::current().block_or_eintr_indefinite(m_write_blocker, &smutex));
 		}
 
 		size_t written = 0;
+		for (; written < buffer.size(); written++)
+			if (!putchar(buffer[written]))
+				break;
+		after_write();
 
-		{
-			SpinLockGuard _(m_write_lock);
-			for (; written < buffer.size(); written++)
-				if (!putchar(buffer[written]))
-					break;
-			after_write();
-		}
-
-		if (can_write_impl())
+		if (can_write())
 			epoll_notify(EPOLLOUT);
 
 		return written;

kernel/kernel/Terminal/TerminalDriver.cpp

@@ -23,6 +23,7 @@ namespace Kernel
 				g_terminal_driver = TRY(TextModeTerminalDriver::create_from_boot_info());
 				break;
 		}
+
 		return {};
 	}
 

kernel/kernel/USB/HID/Keyboard.cpp

@@ -22,6 +22,12 @@ namespace Kernel
 		set_leds(0);
 	}
 
+	BAN::ErrorOr<void> USBKeyboard::initialize()
+	{
+		m_led_region = TRY(DMARegion::create(PAGE_SIZE, PageTable::MemoryType::Normal));
+		return {};
+	}
+
 	void USBKeyboard::start_report()
 	{
 		m_lock_state = m_keyboard_lock.lock();
@@ -190,6 +196,9 @@ namespace Kernel
 	{
 		using KeyModifier = LibInput::KeyEvent::Modifier;
 
+		if (!m_led_region)
+			return;
+
 		size_t report_bits = 0;
 		for (const auto& report : m_outputs)
 		{
@@ -199,40 +208,42 @@ namespace Kernel
 		}
 
 		const size_t report_bytes = (report_bits + 7) / 8;
+		ASSERT(report_bytes <= PAGE_SIZE);
 
-		uint8_t* data = static_cast<uint8_t*>(kmalloc(report_bytes));
-		if (data == nullptr)
-			return;
-		memset(data, 0, report_bytes);
+		PageTable::with_fast_page(m_led_region->paddr(), [&] {
+			uint8_t* data = &PageTable::fast_page_as<uint8_t>();
 
-		size_t bit_offset = 0;
-		for (const auto& report : m_outputs)
-		{
-			if (report.report_id != report_id)
-				continue;
+			memset(data, 0, report_bytes);
 
-			for (size_t i = 0; report.report_size == 1 && i < report.report_count; i++, bit_offset++)
+			size_t bit_offset = 0;
+			for (const auto& report : m_outputs)
 			{
-				const size_t usage = (report.usage_id ? report.usage_id : report.usage_minimum) + bit_offset;
-				switch (usage)
-				{
-					case 0x01:
-						if (mask & KeyModifier::NumLock)
-							data[bit_offset / 8] |= 1u << (bit_offset % 8);
-						break;
-					case 0x02:
-						if (mask & KeyModifier::CapsLock)
-							data[bit_offset / 8] |= 1u << (bit_offset % 8);
-						break;
-					case 0x03:
-						if (mask & KeyModifier::ScrollLock)
-							data[bit_offset / 8] |= 1u << (bit_offset % 8);
-						break;
-				}
-			}
+				if (report.report_id != report_id)
+					continue;
 
-			bit_offset += report.report_size * report.report_count;
-		}
+				for (size_t i = 0; report.report_size == 1 && i < report.report_count; i++, bit_offset++)
+				{
+					const size_t usage = (report.usage_id ? report.usage_id : report.usage_minimum) + bit_offset;
+					switch (usage)
+					{
+						case 0x01:
+							if (mask & KeyModifier::NumLock)
+								data[bit_offset / 8] |= 1u << (bit_offset % 8);
+							break;
+						case 0x02:
+							if (mask & KeyModifier::CapsLock)
+								data[bit_offset / 8] |= 1u << (bit_offset % 8);
+							break;
+						case 0x03:
+							if (mask & KeyModifier::ScrollLock)
+								data[bit_offset / 8] |= 1u << (bit_offset % 8);
+							break;
+					}
+				}
+
+				bit_offset += report.report_size * report.report_count;
+			}
+		});
 
 		USBDeviceRequest request;
 		request.bmRequestType = USB::RequestType::HostToDevice | USB::RequestType::Class | USB::RequestType::Interface;
@@ -240,10 +251,8 @@ namespace Kernel
 		request.wValue = 0x0200 | report_id;
 		request.wIndex = m_driver.interface().descriptor.bInterfaceNumber;
 		request.wLength = report_bytes;
-		if (auto ret = m_driver.device().send_request(request, kmalloc_paddr_of(reinterpret_cast<vaddr_t>(data)).value()); ret.is_error())
+		if (auto ret = m_driver.device().send_request(request, m_led_region->paddr()); ret.is_error())
 			dprintln_if(DEBUG_USB_KEYBOARD, "Failed to update LEDs: {}", ret.error());
-
-		kfree(data);
 	}
 
 	void initialize_scancode_to_keycode()

kernel/kernel/USB/XHCI/Device.cpp

@@ -1,4 +1,3 @@
-#include <BAN/Bitcast.h>
 #include <BAN/ByteSpan.h>
 
 #include <kernel/Lock/LockGuard.h>
@@ -125,8 +124,7 @@ namespace Kernel
 
 		dprintln_if(DEBUG_XHCI, "Retrieving actual max packet size");
 
-		BAN::Vector<uint8_t> buffer;
-		TRY(buffer.resize(8, 0));
+		auto response_region = TRY(DMARegion::create(PAGE_SIZE, PageTable::MemoryType::Normal));
 
 		USBDeviceRequest request;
 		request.bmRequestType = USB::RequestType::DeviceToHost | USB::RequestType::Standard | USB::RequestType::Device;
@@ -134,12 +132,17 @@ namespace Kernel
 		request.wValue        = USB::DescriptorType::DEVICE << 8;
 		request.wIndex        = 0;
 		request.wLength       = 8;
-		TRY(send_request(request, kmalloc_paddr_of((vaddr_t)buffer.data()).value()));
+		TRY(send_request(request, response_region->paddr()));
+
+		uint8_t bMaxPacketSize0;
+		PageTable::with_fast_page(response_region->paddr(), [&bMaxPacketSize0] {
+			bMaxPacketSize0 = PageTable::fast_page_as_sized<uint8_t>(7);
+		});
 
 		dprintln_if(DEBUG_XHCI, "Got device descriptor");
 
 		const bool is_usb3 = (m_speed_class == USB::SpeedClass::SuperSpeed);
-		const uint32_t new_max_packet_size = is_usb3 ? 1u << buffer.back() : buffer.back();
+		const uint32_t new_max_packet_size = is_usb3 ? 1u << bMaxPacketSize0 : bMaxPacketSize0;
 
 		if (m_endpoints[0].max_packet_size == new_max_packet_size)
 			return {};

userspace/libraries/LibC/stdlib.cpp

@@ -780,46 +780,73 @@ static void qsort_swap(void* lhs, void* rhs, size_t width)
 	uint8_t* ulhs = static_cast<uint8_t*>(lhs);
 	uint8_t* urhs = static_cast<uint8_t*>(rhs);
 
-	uint8_t buffer[64];
+	uint8_t temp[64];
 	while (width > 0)
 	{
-		const size_t to_swap = BAN::Math::min(width, sizeof(buffer));
-		memcpy(buffer, ulhs, to_swap);
+		const size_t to_swap = BAN::Math::min(width, sizeof(temp));
+		memcpy(temp, ulhs, to_swap);
 		memcpy(ulhs, urhs, to_swap);
-		memcpy(urhs, buffer, to_swap);
+		memcpy(urhs, temp, to_swap);
 		width -= to_swap;
 		ulhs += to_swap;
 		urhs += to_swap;
 	}
 }
 
-static uint8_t* qsort_partition(uint8_t* pbegin, uint8_t* pend, size_t width, int (*compar)(const void*, const void*))
+struct qsort_pair
 {
-	uint8_t* pivot = pend - width;
-	uint8_t* p1 = pbegin;
-	for (uint8_t* p2 = pbegin; p2 < pivot; p2 += width)
+	uint8_t* lt;
+	uint8_t* gt;
+};
+
+static qsort_pair qsort_partition(uint8_t* pbegin, uint8_t* pend, size_t width, int (*compar)(const void*, const void*))
+{
+	uint8_t* pivot = pbegin + (pend - pbegin) / width / 2 * width;
+
+	uint8_t* lt = pbegin;
+	uint8_t* eq = pbegin;
+	uint8_t* gt = pend;
+
+	while (eq < gt)
 	{
-		if (compar(p2, pivot) >= 0)
-			continue;
-		qsort_swap(p1, p2, width);
-		p1 += width;
+		const int comp = compar(eq, pivot);
+
+		if (comp < 0)
+		{
+			qsort_swap(eq, lt, width);
+			if (pivot == lt)
+				pivot = eq;
+			lt += width;
+			eq += width;
+		}
+		else if (comp > 0)
+		{
+			gt -= width;
+			qsort_swap(eq, gt, width);
+			if (pivot == gt)
+				pivot = eq;
+		}
+		else
+		{
+			eq += width;
+		}
 	}
-	qsort_swap(p1, pivot, width);
-	return p1;
+
+	return { lt, gt };
 }
 
 static void qsort_impl(uint8_t* pbegin, uint8_t* pend, size_t width, int (*compar)(const void*, const void*))
 {
-	if ((pend - pbegin) / width <= 1)
+	if (pbegin + width >= pend)
 		return;
-	uint8_t* mid = qsort_partition(pbegin, pend, width, compar);
-	qsort_impl(pbegin, mid, width, compar);
-	qsort_impl(mid + width, pend, width, compar);
+	auto [lt, gt] = qsort_partition(pbegin, pend, width, compar);
+	qsort_impl(pbegin, lt, width, compar);
+	qsort_impl(gt, pend, width, compar);
 }
 
 void qsort(void* base, size_t nel, size_t width, int (*compar)(const void*, const void*))
 {
-	if (width == 0)
+	if (width == 0 || nel <= 1)
 		return;
 	uint8_t* pbegin = static_cast<uint8_t*>(base);
 	qsort_impl(pbegin, pbegin + nel * width, width, compar);

userspace/libraries/LibC/unistd.cpp

@@ -1,5 +1,6 @@
 #include <BAN/Assert.h>
 #include <BAN/Debug.h>
+#include <BAN/ScopeGuard.h>
 #include <BAN/StringView.h>
 
 #include <LibELF/AuxiliaryVector.h>
@@ -208,6 +209,18 @@ static void __dump_backtrace(int sig, siginfo_t*, void* context)
 			return "unknown signal";
 		};
 
+	// NOTE: we cannot use stddbg as that is not async-signal-safe.
+	//       POSIX says dprintf isn't either but our implementation is!
+
+	int fd = open("/dev/debug", O_WRONLY);
+	if (fd == -1)
+	{
+		perror("failed to open debug device for backtrace");
+		return;
+	}
+
+	dprintf(fd, "received %s, backtrace:\n", signal_name(sig));
+
 	const auto* ucontext = static_cast<ucontext_t*>(context);
 #if defined(__x86_64__)
 	const uintptr_t stack_base = ucontext->uc_mcontext.gregs[REG_RBP];
@@ -225,18 +238,6 @@ static void __dump_backtrace(int sig, siginfo_t*, void* context)
 
 	const auto* stackframe = reinterpret_cast<struct stackframe*>(stack_base);
 
-	// NOTE: we cannot use stddbf as that is not async-signal-safe.
-	//       POSIX says dprintf isn't either but our implementation is!
-
-	int fd = open("/dev/debug", O_WRONLY);
-	if (fd == -1)
-	{
-		perror("failed to open debug device for backtrace");
-		return;
-	}
-
-	dprintf(fd, "received %s, backtrace:\n", signal_name(sig));
-
 	__dump_symbol(fd, reinterpret_cast<void*>(instruction));
 	for (size_t i = 0; i < 128 && stackframe; i++)
 	{
@@ -436,18 +437,20 @@ static int exec_impl(const char* pathname, char* const* argv, char* const* envp,
 static int exec_impl_shebang(FILE* fp, const char* pathname, char* const* argv, char* const* envp, int shebang_depth)
 {
 	constexpr size_t buffer_len = PATH_MAX + 1 + ARG_MAX + 1;
+
 	char* buffer = static_cast<char*>(malloc(buffer_len));
 	if (buffer == nullptr)
 	{
 		fclose(fp);
 		return -1;
 	}
+	BAN::ScopeGuard _1([buffer] { free(buffer); });
 
-	if (fgets(buffer, buffer_len, fp) == nullptr)
-	{
-		free(buffer);
+	buffer = fgets(buffer, buffer_len, fp);
+	fclose(fp);
+
+	if (buffer == nullptr)
 		return -1;
-	}
 
 	const auto sv_trim_whitespace =
 		[](BAN::StringView sv) -> BAN::StringView
@@ -462,7 +465,6 @@ static int exec_impl_shebang(FILE* fp, const char* pathname, char* const* argv,
 	BAN::StringView buffer_sv = buffer;
 	if (buffer_sv.back() != '\n')
 	{
-		free(buffer);
 		errno = ENOEXEC;
 		return -1;
 	}
@@ -479,7 +481,6 @@ static int exec_impl_shebang(FILE* fp, const char* pathname, char* const* argv,
 
 	if (interpreter.empty())
 	{
-		free(buffer);
 		errno = ENOEXEC;
 		return -1;
 	}
@@ -496,22 +497,19 @@ static int exec_impl_shebang(FILE* fp, const char* pathname, char* const* argv,
 	const size_t extra_args = 1 + !argument.empty();
 	char** new_argv = static_cast<char**>(malloc((extra_args + old_argc + 1) * sizeof(char*)));;
 	if (new_argv == nullptr)
-	{
-		free(buffer);
 		return -1;
-	}
+	BAN::ScopeGuard _2([new_argv] { free(new_argv); });
 
 	new_argv[0] = const_cast<char*>(pathname);
 	if (!argument.empty())
 		new_argv[1] = const_cast<char*>(argument.data());
-	for (size_t i = 0; i < old_argc; i++)
-		new_argv[i + extra_args] = argv[i];
-	new_argv[old_argc + extra_args] = nullptr;
+	if (old_argc)
+		new_argv[extra_args] = const_cast<char*>(pathname);
+	for (size_t i = 1; i < old_argc; i++)
+		new_argv[extra_args + i] = argv[i];
+	new_argv[extra_args + old_argc] = nullptr;
 
-	exec_impl(interpreter.data(), new_argv, envp, true, shebang_depth + 1);
-	free(new_argv);
-	free(buffer);
-	return -1;
+	return exec_impl(interpreter.data(), new_argv, envp, true, shebang_depth + 1);
 }
 
 static int execl_impl(const char* pathname, const char* arg0, va_list ap, bool has_env, bool do_path_resolution)

userspace/programs/Shell/Builtin.cpp

@@ -216,9 +216,10 @@ void Builtin::initialize()
 					for (const auto& path_dir : path_dirs)
 					{
 						char path_buffer[PATH_MAX];
-						memcpy(path_buffer,                   path_dir.data(), path_dir.size());
-						memcpy(path_buffer + path_dir.size(), argument.data(), argument.size());
-						path_buffer[path_dir.size() + argument.size()] = '\0';
+						memcpy(path_buffer, path_dir.data(), path_dir.size());
+						path_buffer[path_dir.size()] = '/';
+						memcpy(path_buffer + path_dir.size() + 1, argument.data(), argument.size());
+						path_buffer[path_dir.size() + 1 + argument.size()] = '\0';
 
 						if (is_executable_file(path_buffer))
 						{

userspace/programs/dirname/main.cpp

@@ -1,18 +1,56 @@
+#include <getopt.h>
 #include <libgen.h>
 #include <stdio.h>
 
-int usage(const char* argv0, int ret)
+int main(int argc, char* argv[])
 {
-	FILE* fout = ret ? stderr : stdout;
-	fprintf(fout, "usage: %s STRING\n", argv0);
-	return ret;
-}
+	bool zero = false;
+
+	for (;;)
+	{
+		static option long_options[] {
+			{ "zero", no_argument, nullptr, 'z' },
+			{ "help", no_argument, nullptr, 'h' },
+		};
+
+		int ch = getopt_long(argc, argv, "zh", long_options, nullptr);
+		if (ch == -1)
+			break;
+
+		switch (ch)
+		{
+			case 'z':
+				zero = true;
+				break;
+			case 'h':
+				fprintf(stderr, "usage: %s [OPTIONS]...\n", argv[0]);
+				fprintf(stderr, "  control the audio server\n");
+				fprintf(stderr, "OPTIONS:\n");
+				fprintf(stderr, "  -l, --list      list devices and their pins\n");
+				fprintf(stderr, "  -d, --device N  set device index N as the current one\n");
+				fprintf(stderr, "  -p, --pin N     set pin N as the current one\n");
+				fprintf(stderr, "  -v, --volume N  set volume to N%%. if + or - is given, volume is relative to the current volume\n");
+				fprintf(stderr, "  -h, --help      show this message and exit\n");
+				return 0;
+			case '?':
+				fprintf(stderr, "invalid option %c\n", optopt);
+				fprintf(stderr, "see '%s --help' for usage\n", argv[0]);
+				return 1;
+		}
+	}
+
+	if (optind >= argc)
+	{
+		fprintf(stderr, "missing operand\n");
+		fprintf(stderr, "see '%s --help' for usage\n", argv[0]);
+		return 1;
+	}
+
+	for (int i = optind; i < argc; i++)
+	{
+		printf("%s", dirname(argv[i]));
+		putchar(zero ? '\0' : '\n');
+	}
 
-int main(int argc, const char* argv[])
-{
-	if (argc != 2)
-		return usage(argv[0], 1);
-	const char* result = dirname(const_cast<char*>(argv[1]));
-	printf("%s\n", result);
 	return 0;
 }

userspace/tests/test-sort/main.cpp

@@ -39,6 +39,32 @@ bool is_sorted(BAN::Vector<T>& vec)
 		}																\
 	} while (0)
 
+
+#define TEST_ALGORITHM_RADIX(ms) do {									\
+		uint64_t duration_us = 0;										\
+		printf("radix with preallocated buffer\n");						\
+		for (size_t size = 100; duration_us < ms * 1000; size *= 10) {	\
+			BAN::Vector<unsigned> data(size, 0);						\
+			for (auto& val : data)										\
+				val = rand() % 100;										\
+			BAN::Vector<unsigned> temp(size);							\
+			uint64_t start_ns = CURRENT_NS();							\
+			BAN::sort::radix_sort(data.begin(), data.end(), temp.span()); \
+			uint64_t stop_ns = CURRENT_NS();							\
+			if (!is_sorted(data)) {										\
+				printf("  \e[31mFAILED!\e[m\n");						\
+				break;													\
+			}															\
+			duration_us = (stop_ns - start_ns) / 1'000;					\
+			printf("  %5d.%03d ms (%zu)\n",								\
+				(int)(duration_us / 1000),								\
+				(int)(duration_us % 1000),								\
+				size													\
+			);															\
+		}																\
+	} while (0)
+
+
 #define TEST_ALGORITHM_QSORT(ms) do {									\
 		uint64_t duration_us = 0;										\
 		printf("qsort\n");												\
@@ -72,5 +98,6 @@ int main()
 	TEST_ALGORITHM(100,  BAN::sort::intro_sort);
 	TEST_ALGORITHM(1000, BAN::sort::sort);
 	TEST_ALGORITHM(1000, BAN::sort::radix_sort);
+	TEST_ALGORITHM_RADIX(1000);
 	TEST_ALGORITHM_QSORT(100);
 }