Kernel: Fix most of mutex + block race conditions

All block functions now take an optional mutex parameter that is
atomically unlocked instead of having the user unlock it before hand.
This prevents a ton of race conditions everywhere in the code!

kernel/include/kernel/FS/Inode.h

@@ -182,7 +182,7 @@ namespace Kernel
 
 	private:
 		BAN::WeakPtr<SharedFileData> m_shared_region;
-		Mutex m_epoll_mutex;
+		SpinLock m_epoll_lock;
 		BAN::LinkedList<class Epoll*> m_epolls;
 		friend class Epoll;
 		friend class FileBackedRegion;

kernel/include/kernel/Lock/LockGuard.h

@@ -29,30 +29,4 @@ namespace Kernel
 		Lock& m_lock;
 	};
 
-	template<typename Lock>
-	class LockFreeGuard
-	{
-		BAN_NON_COPYABLE(LockFreeGuard);
-		BAN_NON_MOVABLE(LockFreeGuard);
-
-	public:
-		LockFreeGuard(Lock& lock)
-			: m_lock(lock)
-			, m_depth(lock.lock_depth())
-		{
-			for (uint32_t i = 0; i < m_depth; i++)
-				m_lock.unlock();
-		}
-
-		~LockFreeGuard()
-		{
-			for (uint32_t i = 0; i < m_depth; i++)
-				m_lock.lock();
-		}
-
-	private:
-		Lock& m_lock;
-		const uint32_t m_depth;
-	};
-
 }

kernel/include/kernel/Lock/Mutex.h

@@ -9,7 +9,19 @@
 namespace Kernel
 {
 
-	class Mutex
+	class BaseMutex
+	{
+	public:
+		virtual void lock() = 0;
+		virtual bool try_lock() = 0;
+		virtual void unlock() = 0;
+
+		virtual pid_t locker() const = 0;
+		virtual bool is_locked() const = 0;
+		virtual uint32_t lock_depth() const = 0;
+	};
+
+	class Mutex : public BaseMutex
 	{
 		BAN_NON_COPYABLE(Mutex);
 		BAN_NON_MOVABLE(Mutex);
@@ -17,9 +29,10 @@ namespace Kernel
 	public:
 		Mutex() = default;
 
-		void lock()
+		void lock() override
 		{
 			const auto tid = Thread::current_tid();
+			ASSERT(!tid || !Thread::current().has_spinlock());
 			if (tid == m_locker)
 				ASSERT(m_lock_depth > 0);
 			else
@@ -37,9 +50,10 @@ namespace Kernel
 			m_lock_depth++;
 		}
 
-		bool try_lock()
+		bool try_lock() override
 		{
 			const auto tid = Thread::current_tid();
+			ASSERT(!tid || !Thread::current().has_spinlock());
 			if (tid == m_locker)
 				ASSERT(m_lock_depth > 0);
 			else
@@ -55,7 +69,7 @@ namespace Kernel
 			return true;
 		}
 
-		void unlock()
+		void unlock() override
 		{
 			const auto tid = Thread::current_tid();
 			ASSERT(m_locker == tid);
@@ -68,16 +82,16 @@ namespace Kernel
 			}
 		}
 
-		pid_t locker() const { return m_locker; }
-		bool is_locked() const { return m_locker != -1; }
-		uint32_t lock_depth() const { return m_lock_depth; }
+		pid_t locker() const override { return m_locker; }
+		bool is_locked() const override { return m_locker != -1; }
+		uint32_t lock_depth() const override { return m_lock_depth; }
 
 	private:
 		BAN::Atomic<pid_t>	m_locker		{ -1 };
 		uint32_t			m_lock_depth	{  0 };
 	};
 
-	class PriorityMutex
+	class PriorityMutex : public BaseMutex
 	{
 		BAN_NON_COPYABLE(PriorityMutex);
 		BAN_NON_MOVABLE(PriorityMutex);
@@ -85,7 +99,7 @@ namespace Kernel
 	public:
 		PriorityMutex() = default;
 
-		void lock()
+		void lock() override
 		{
 			const auto tid = Thread::current_tid();
 			ASSERT(!tid || !Thread::current().has_spinlock());
@@ -110,7 +124,7 @@ namespace Kernel
 			m_lock_depth++;
 		}
 
-		bool try_lock()
+		bool try_lock() override
 		{
 			const auto tid = Thread::current_tid();
 			ASSERT(!tid || !Thread::current().has_spinlock());
@@ -133,7 +147,7 @@ namespace Kernel
 			return true;
 		}
 
-		void unlock()
+		void unlock() override
 		{
 			const auto tid = Thread::current_tid();
 			ASSERT(m_locker == tid);
@@ -149,9 +163,9 @@ namespace Kernel
 			}
 		}
 
-		pid_t locker() const { return m_locker; }
-		bool is_locked() const { return m_locker != -1; }
-		uint32_t lock_depth() const { return m_lock_depth; }
+		pid_t locker() const override { return m_locker; }
+		bool is_locked() const override { return m_locker != -1; }
+		uint32_t lock_depth() const override { return m_lock_depth; }
 
 	private:
 		BAN::Atomic<pid_t>		m_locker		{ -1 };

kernel/include/kernel/Lock/SpinLock.h

@@ -24,6 +24,8 @@ namespace Kernel
 
 		void unlock(InterruptState state);
 
+		uint32_t lock_depth() const { return current_processor_has_lock(); }
+
 		bool current_processor_has_lock() const
 		{
 			return m_locker.load(BAN::MemoryOrder::memory_order_relaxed) == Processor::current_id().as_u32();
@@ -72,6 +74,8 @@ namespace Kernel
 			Processor::set_interrupt_state(state);
 		}
 
+		uint32_t lock_depth() const { return m_lock_depth; }
+
 		bool current_processor_has_lock() const
 		{
 			return m_locker.load(BAN::MemoryOrder::memory_order_relaxed) == Processor::current_id().as_u32();
@@ -82,6 +86,9 @@ namespace Kernel
 		uint32_t                             m_lock_depth { 0 };
 	};
 
+	template<typename Lock>
+	class SpinLockGuardAsMutex;
+
 	template<typename Lock>
 	class SpinLockGuard
 	{
@@ -103,6 +110,7 @@ namespace Kernel
 	private:
 		Lock& m_lock;
 		InterruptState m_state;
+		friend class SpinLockGuardAsMutex<Lock>;
 	};
 
 }

kernel/include/kernel/Memory/MemoryRegion.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <BAN/UniqPtr.h>
+#include <kernel/Lock/Mutex.h>
 #include <kernel/Memory/PageTable.h>
 #include <kernel/Memory/Types.h>
 #include <kernel/ThreadBlocker.h>
@@ -41,9 +42,9 @@ namespace Kernel
 		size_t virtual_page_count() const { return BAN::Math::div_round_up<size_t>(m_size, PAGE_SIZE); }
 		size_t physical_page_count() const { return m_physical_page_count; }
 
-		void pin() { m_pinned_count++; }
-		void unpin() { if (--m_pinned_count == 0) m_pinned_blocker.unblock(); }
-		void wait_not_pinned() { while (m_pinned_count) m_pinned_blocker.block_with_timeout_ms(100); }
+		void pin();
+		void unpin();
+		void wait_not_pinned();
 
 		virtual BAN::ErrorOr<void> msync(vaddr_t, size_t, int) = 0;
 
@@ -68,6 +69,7 @@ namespace Kernel
 		vaddr_t m_vaddr { 0 };
 		size_t m_physical_page_count { 0 };
 
+		Mutex m_pinned_mutex;
 		BAN::Atomic<size_t> m_pinned_count { 0 };
 		ThreadBlocker m_pinned_blocker;
 	};

kernel/include/kernel/Process.h

@@ -331,7 +331,6 @@ namespace Kernel
 
 		bool m_is_userspace { false };
 
-		SpinLock m_child_exit_lock;
 		BAN::Vector<ChildExitStatus> m_child_exit_statuses;
 		ThreadBlocker m_child_exit_blocker;
 

kernel/include/kernel/Scheduler.h

@@ -11,6 +11,7 @@
 namespace Kernel
 {
 
+	class BaseMutex;
 	class Thread;
 	class ThreadBlocker;
 
@@ -86,7 +87,7 @@ namespace Kernel
 		// if thread is already bound, this will never fail
 		BAN::ErrorOr<void> add_thread(Thread*);
 
-		void block_current_thread(ThreadBlocker* thread_blocker, uint64_t wake_time_ns);
+		void block_current_thread(ThreadBlocker* thread_blocker, uint64_t wake_time_ns, BaseMutex* mutex);
 		void unblock_thread(Thread*);
 
 		Thread& current_thread();

kernel/include/kernel/Thread.h

@@ -58,13 +58,27 @@ namespace Kernel
 		bool add_signal(int signal);
 
 		// blocks current thread and returns either on unblock, eintr, spuriously or after timeout
-		BAN::ErrorOr<void> sleep_or_eintr_ms(uint64_t ms) { ASSERT(!BAN::Math::will_multiplication_overflow<uint64_t>(ms, 1'000'000)); return sleep_or_eintr_ns(ms * 1'000'000); }
+		// if mutex is not nullptr, it will be atomically freed before blocking and automatically locked on wake
 		BAN::ErrorOr<void> sleep_or_eintr_ns(uint64_t ns);
-		BAN::ErrorOr<void> block_or_eintr_indefinite(ThreadBlocker& thread_blocker);
-		BAN::ErrorOr<void> block_or_eintr_or_timeout_ms(ThreadBlocker& thread_blocker, uint64_t timeout_ms, bool etimedout) { ASSERT(!BAN::Math::will_multiplication_overflow<uint64_t>(timeout_ms, 1'000'000)); return block_or_eintr_or_timeout_ns(thread_blocker, timeout_ms * 1'000'000, etimedout); }
-		BAN::ErrorOr<void> block_or_eintr_or_waketime_ms(ThreadBlocker& thread_blocker, uint64_t wake_time_ms, bool etimedout) { ASSERT(!BAN::Math::will_multiplication_overflow<uint64_t>(wake_time_ms, 1'000'000)); return block_or_eintr_or_waketime_ns(thread_blocker, wake_time_ms * 1'000'000, etimedout); }
-		BAN::ErrorOr<void> block_or_eintr_or_timeout_ns(ThreadBlocker& thread_blocker, uint64_t timeout_ns, bool etimedout);
-		BAN::ErrorOr<void> block_or_eintr_or_waketime_ns(ThreadBlocker& thread_blocker, uint64_t wake_time_ns, bool etimedout);
+		BAN::ErrorOr<void> block_or_eintr_indefinite(ThreadBlocker& thread_blocker, BaseMutex* mutex);
+		BAN::ErrorOr<void> block_or_eintr_or_timeout_ns(ThreadBlocker& thread_blocker, uint64_t timeout_ns, bool etimedout, BaseMutex* mutex);
+		BAN::ErrorOr<void> block_or_eintr_or_waketime_ns(ThreadBlocker& thread_blocker, uint64_t wake_time_ns, bool etimedout, BaseMutex* mutex);
+
+		BAN::ErrorOr<void> sleep_or_eintr_ms(uint64_t ms)
+		{
+			ASSERT(!BAN::Math::will_multiplication_overflow<uint64_t>(ms, 1'000'000));
+			return sleep_or_eintr_ns(ms * 1'000'000);
+		}
+		BAN::ErrorOr<void> block_or_eintr_or_timeout_ms(ThreadBlocker& thread_blocker, uint64_t timeout_ms, bool etimedout, BaseMutex* mutex)
+		{
+			ASSERT(!BAN::Math::will_multiplication_overflow<uint64_t>(timeout_ms, 1'000'000));
+			return block_or_eintr_or_timeout_ns(thread_blocker, timeout_ms * 1'000'000, etimedout, mutex);
+		}
+		BAN::ErrorOr<void> block_or_eintr_or_waketime_ms(ThreadBlocker& thread_blocker, uint64_t wake_time_ms, bool etimedout, BaseMutex* mutex)
+		{
+			ASSERT(!BAN::Math::will_multiplication_overflow<uint64_t>(wake_time_ms, 1'000'000));
+			return block_or_eintr_or_waketime_ns(thread_blocker, wake_time_ms * 1'000'000, etimedout, mutex);
+		}
 
 		pid_t tid() const { return m_tid; }
 

kernel/include/kernel/ThreadBlocker.h

@@ -10,13 +10,23 @@ namespace Kernel
 	class ThreadBlocker
 	{
 	public:
-		void block_indefinite();
-		void block_with_timeout_ms(uint64_t timeout_ms) { ASSERT(!BAN::Math::will_multiplication_overflow<uint64_t>(timeout_ms, 1'000'000)); return block_with_timeout_ns(timeout_ms * 1'000'000); }
-		void block_with_wake_time_ms(uint64_t wake_time_ms) { ASSERT(!BAN::Math::will_multiplication_overflow<uint64_t>(wake_time_ms, 1'000'000)); return block_with_wake_time_ns(wake_time_ms * 1'000'000); }
-		void block_with_timeout_ns(uint64_t timeout_ns);
-		void block_with_wake_time_ns(uint64_t wake_time_ns);
+		void block_indefinite(BaseMutex*);
+		void block_with_timeout_ns(uint64_t timeout_ns, BaseMutex*);
+		void block_with_wake_time_ns(uint64_t wake_time_ns, BaseMutex*);
 		void unblock();
 
+
+		void block_with_timeout_ms(uint64_t timeout_ms, BaseMutex* mutex)
+		{
+			ASSERT(!BAN::Math::will_multiplication_overflow<uint64_t>(timeout_ms, 1'000'000));
+			return block_with_timeout_ns(timeout_ms * 1'000'000, mutex);
+		}
+		void block_with_wake_time_ms(uint64_t wake_time_ms, BaseMutex* mutex)
+		{
+			ASSERT(!BAN::Math::will_multiplication_overflow<uint64_t>(wake_time_ms, 1'000'000));
+			return block_with_wake_time_ns(wake_time_ms * 1'000'000, mutex);
+		}
+
 	private:
 		void add_thread_to_block_queue(SchedulerQueue::Node*);
 		void remove_blocked_thread(SchedulerQueue::Node*);