Kernel: Optimize futexes

Add support for processor local futexes. These work the exact same way
as global ones, but only lock a process specific lock and use a process
specific hash map.

Also reduce the time futex lock is held. There was no need to hold the
global lock while validating addresses in the process' address space.

kernel/include/kernel/Process.h

@@ -346,6 +346,19 @@ namespace Kernel
 
 		vaddr_t m_shared_page_vaddr { 0 };
 
+		struct futex_t
+		{
+			ThreadBlocker blocker;
+			uint32_t waiters { 0 };
+			uint32_t to_wakeup { 0 };
+		};
+
+		static BAN::HashMap<paddr_t, BAN::UniqPtr<futex_t>> s_futexes;
+		static Mutex s_futex_lock;
+
+		BAN::HashMap<paddr_t, BAN::UniqPtr<futex_t>> m_futexes;
+		Mutex m_futex_lock;
+
 		BAN::Vector<Thread*> m_threads;
 
 		struct pthread_info_t

kernel/kernel/Process.cpp

@@ -38,14 +38,8 @@ namespace Kernel
 	static BAN::Vector<Process*> s_processes;
 	static RecursiveSpinLock s_process_lock;
 
-	struct futex_t
+	BAN::HashMap<paddr_t, BAN::UniqPtr<Process::futex_t>> Process::s_futexes;
-	{
+	Mutex Process::s_futex_lock;
-		ThreadBlocker blocker;
-		uint32_t waiters { 0 };
-		uint32_t to_wakeup { 0 };
-	};
-	static BAN::HashMap<paddr_t, BAN::UniqPtr<futex_t>> s_futexes;
-	static Mutex s_futex_lock;
 
 	static void for_each_process(const BAN::Function<BAN::Iteration(Process&)>& callback)
 	{
@@ -3107,12 +3101,9 @@ namespace Kernel
 			return BAN::Error::from_errno(EINVAL);
 
 		const bool is_realtime = (op & FUTEX_REALTIME);
+		const bool is_private = (op & FUTEX_PRIVATE);
 		op &= ~(FUTEX_PRIVATE | FUTEX_REALTIME);
 
-		// TODO: possibly optimize private futexes?
-
-		LockGuard _(s_futex_lock);
-
 		auto* buffer_region = TRY(validate_and_pin_pointer_access(addr, sizeof(uint32_t), false));
 		BAN::ScopeGuard pin_guard([&] { if (buffer_region) buffer_region->unpin(); });
 
@@ -3122,17 +3113,26 @@ namespace Kernel
 		switch (op)
 		{
 			case FUTEX_WAIT:
-			{
+				break;
-				if (BAN::atomic_load(*addr) != val)
+			case FUTEX_WAKE:
-					return BAN::Error::from_errno(EAGAIN);
+				abstime = nullptr;
+				break;
+			default:
+				return BAN::Error::from_errno(ENOSYS);
+		}
 
 		const uint64_t wake_time_ns =
 			TRY([abstime, is_realtime, this]() -> BAN::ErrorOr<uint64_t>
 			{
 				if (abstime == nullptr)
 					return BAN::numeric_limits<uint64_t>::max();
+				const uint64_t abs_ns =
+					TRY([abstime, this]() -> BAN::ErrorOr<uint64_t>
+					{
+						LockGuard _(m_process_lock);
 						TRY(validate_pointer_access(abstime, sizeof(*abstime), false));
-						const uint64_t abs_ns = abstime->tv_sec * 1'000'000'000 + abstime->tv_nsec;
+						return abstime->tv_sec * 1'000'000'000 + abstime->tv_nsec;
+					}());
 				if (!is_realtime)
 					return abs_ns;
 				const auto realtime = SystemTimer::get().real_time();
@@ -3142,20 +3142,32 @@ namespace Kernel
 				return SystemTimer::get().ns_since_boot() + (abs_ns - real_ns);
 			}());
 
-				auto it = s_futexes.find(paddr);
+		auto& futex_lock = is_private ? m_futex_lock : s_futex_lock;
-				if (it == s_futexes.end())
+		auto& futexes = is_private ? m_futexes : s_futexes;
-					it = TRY(s_futexes.emplace(paddr, TRY(BAN::UniqPtr<futex_t>::create())));
+
+		LockGuard _(futex_lock);
+
+		switch (op)
+		{
+			case FUTEX_WAIT:
+			{
+				if (BAN::atomic_load(*addr) != val)
+					return BAN::Error::from_errno(EAGAIN);
+
+				auto it = futexes.find(paddr);
+				if (it == futexes.end())
+					it = TRY(futexes.emplace(paddr, TRY(BAN::UniqPtr<futex_t>::create())));
 				futex_t* const futex = it->value.ptr();
 
 				futex->waiters++;
-				BAN::ScopeGuard _([futex, paddr] {
+				BAN::ScopeGuard _([&futexes, futex, paddr] {
 					if (--futex->waiters == 0)
-						s_futexes.remove(paddr);
+						futexes.remove(paddr);
 				});
 
 				for (;;)
 				{
-					TRY(Thread::current().block_or_eintr_or_waketime_ns(futex->blocker, wake_time_ns, true, &s_futex_lock));
+					TRY(Thread::current().block_or_eintr_or_waketime_ns(futex->blocker, wake_time_ns, true, &futex_lock));
 					if (BAN::atomic_load(*addr) == val || futex->to_wakeup == 0)
 						continue;
 					futex->to_wakeup--;
@@ -3164,8 +3176,8 @@ namespace Kernel
 			}
 			case FUTEX_WAKE:
 			{
-				auto it = s_futexes.find(paddr);
+				auto it = futexes.find(paddr);
-				if (it == s_futexes.end())
+				if (it == futexes.end())
 					return 0;
 				futex_t* const futex = it->value.ptr();
 
@@ -3178,8 +3190,6 @@ namespace Kernel
 				futex->blocker.unblock();
 				return 0;
 			}
-			default:
-				return BAN::Error::from_errno(ENOSYS);
 		}
 
 		ASSERT_NOT_REACHED();

userspace/libraries/LibC/pthread.cpp

@@ -884,9 +884,11 @@ int pthread_mutex_unlock(pthread_mutex_t* mutex)
 	mutex->lock_depth--;
 	if (mutex->lock_depth == 0)
 	{
+		const int op = FUTEX_WAKE | (mutex->attr.shared ? 0 : FUTEX_PRIVATE);
+
 		BAN::atomic_store(mutex->futex, 0, BAN::memory_order_release);
 		if (BAN::atomic_load(mutex->waiters))
-			futex(FUTEX_WAKE, &mutex->futex, 1, nullptr);
+			futex(op, &mutex->futex, 1, nullptr);
 	}
 
 	return 0;

userspace/libraries/LibC/semaphore.cpp

@@ -28,8 +28,9 @@ int sem_getvalue(sem_t* __restrict sem, int* __restrict sval)
 int sem_post(sem_t* sem)
 {
 	const auto old = BAN::atomic_fetch_add(sem->value, 1);
+	const int op = FUTEX_WAKE | (sem->shared ? 0 : FUTEX_PRIVATE);
 	if (old == 0)
-		futex(FUTEX_WAKE, &sem->value, 1, nullptr);
+		futex(op, &sem->value, 1, nullptr);
 	return 0;
 }