From 9f4b451501bad8ed7adb478df409bf2f0f7c96bf Mon Sep 17 00:00:00 2001
From: Bananymous <oskari.alaranta@bananymous.com>
Date: Thu, 29 May 2025 22:08:24 +0300
Subject: [PATCH] Kernel: Fix epoll deadlock

If epoll_wait and epoll_notify were called at the same time, there was a
possible deadlock when epoll was confirming the event from the inode
---
 kernel/include/kernel/FS/Inode.h |  1 +
 kernel/kernel/Epoll.cpp          | 14 ++++++++++++++
 2 files changed, 15 insertions(+)

diff --git a/kernel/include/kernel/FS/Inode.h b/kernel/include/kernel/FS/Inode.h
index 833d0342..9ba4088f 100644
--- a/kernel/include/kernel/FS/Inode.h
+++ b/kernel/include/kernel/FS/Inode.h
@@ -184,6 +184,7 @@ namespace Kernel
 		BAN::WeakPtr<SharedFileData> m_shared_region;
 		Mutex m_epoll_mutex;
 		BAN::LinkedList<class Epoll*> m_epolls;
+		friend class Epoll;
 		friend class FileBackedRegion;
 		friend class OpenFileDescriptorSet;
 		friend class SharedFileData;
diff --git a/kernel/kernel/Epoll.cpp b/kernel/kernel/Epoll.cpp
index a88fcba8..49a9eeb1 100644
--- a/kernel/kernel/Epoll.cpp
+++ b/kernel/kernel/Epoll.cpp
@@ -60,6 +60,8 @@ namespace Kernel
 
 		for (;;)
 		{
+			bool failed_lock = false;
+
 			{
 				LockGuard _(m_mutex);
 				for (auto it = m_ready_events.begin(); it != m_ready_events.end() && count < event_span.size();)
@@ -70,6 +72,14 @@ namespace Kernel
 					const uint32_t listen_mask = (listen.events & (EPOLLIN | EPOLLOUT)) | EPOLLERR | EPOLLHUP;
 
 					events &= listen_mask;
+
+					// This prevents a possible deadlock
+					if (!inode->m_mutex.try_lock())
+					{
+						failed_lock = true;
+						continue;
+					}
+
 #define CHECK_EVENT_BIT(mask, func) \
 					if ((events & mask) && !inode->func()) \
 						events &= ~mask;
@@ -79,6 +89,8 @@ namespace Kernel
 					CHECK_EVENT_BIT(EPOLLHUP, has_hungup);
 #undef CHECK_EVENT_BIT
 
+					inode->m_mutex.unlock();
+
 					if (events == 0)
 					{
 						m_ready_events.remove(it);
@@ -107,6 +119,8 @@ namespace Kernel
 			const uint64_t current_ns = SystemTimer::get().ns_since_boot();
 			if (current_ns >= waketime_ns)
 				break;
+			if (failed_lock)
+				continue;
 			const uint64_t timeout_ns = BAN::Math::min<uint64_t>(100'000'000, waketime_ns - current_ns);
 			TRY(Thread::current().block_or_eintr_or_timeout_ns(m_thread_blocker, timeout_ns, false));
 		}