Kernel: Fix all broken locks from new mutexes

This commit is contained in:
2024-02-28 22:39:02 +02:00
parent 1971813336
commit d94f6388b7
66 changed files with 681 additions and 647 deletions

View File

@@ -1,32 +1,30 @@
#pragma once
#include <BAN/Formatter.h>
#include <kernel/Lock/SpinLock.h>
#define dprintln(...) \
do { \
Debug::DebugLock::lock(); \
Kernel::SpinLockGuard _(Debug::s_debug_lock); \
Debug::print_prefix(__FILE__, __LINE__); \
BAN::Formatter::print(Debug::putchar, __VA_ARGS__); \
BAN::Formatter::print(Debug::putchar, "\r\n"); \
Debug::DebugLock::unlock(); \
} while(false)
#define dwarnln(...) \
do { \
Debug::DebugLock::lock(); \
Kernel::SpinLockGuard _(Debug::s_debug_lock); \
BAN::Formatter::print(Debug::putchar, "\e[33m"); \
dprintln(__VA_ARGS__); \
BAN::Formatter::print(Debug::putchar, "\e[m"); \
Debug::DebugLock::unlock(); \
} while(false)
#define derrorln(...) \
do { \
Debug::DebugLock::lock(); \
Kernel::SpinLockGuard _(Debug::s_debug_lock); \
BAN::Formatter::print(Debug::putchar, "\e[31m"); \
dprintln(__VA_ARGS__); \
BAN::Formatter::print(Debug::putchar, "\e[m"); \
Debug::DebugLock::unlock(); \
} while(false)
#define dprintln_if(cond, ...) \
@@ -55,10 +53,5 @@ namespace Debug
void putchar(char);
void print_prefix(const char*, int);
class DebugLock
{
public:
static void lock();
static void unlock();
};
extern Kernel::RecursiveSpinLock s_debug_lock;
}

View File

@@ -3,6 +3,7 @@
#include <BAN/Vector.h>
#include <kernel/Device/Device.h>
#include <kernel/FS/TmpFS/FileSystem.h>
#include <kernel/Lock/Mutex.h>
#include <kernel/Semaphore.h>
namespace Kernel
@@ -29,7 +30,7 @@ namespace Kernel
{ }
private:
mutable SpinLock m_device_lock;
mutable Mutex m_device_lock;
BAN::Vector<BAN::RefPtr<Device>> m_devices;

View File

@@ -106,7 +106,7 @@ namespace Kernel
};
private:
RecursiveSpinLock m_lock;
Mutex m_mutex;
BAN::RefPtr<BlockDevice> m_block_device;

View File

@@ -9,7 +9,8 @@
#include <kernel/API/DirectoryEntry.h>
#include <kernel/Credentials.h>
#include <kernel/SpinLock.h>
#include <kernel/Debug.h>
#include <kernel/Lock/Mutex.h>
#include <sys/socket.h>
#include <sys/types.h>
@@ -157,7 +158,7 @@ namespace Kernel
virtual BAN::ErrorOr<long> ioctl_impl(int request, void* arg) { return BAN::Error::from_errno(ENOTSUP); }
protected:
mutable RecursivePrioritySpinLock m_lock;
mutable PriorityMutex m_mutex;
private:
BAN::WeakPtr<SharedFileData> m_shared_region;

View File

@@ -2,7 +2,6 @@
#include <kernel/FS/Inode.h>
#include <kernel/Semaphore.h>
#include <kernel/SpinLock.h>
namespace Kernel
{

View File

@@ -4,9 +4,8 @@
#include <BAN/Iteration.h>
#include <kernel/FS/FileSystem.h>
#include <kernel/FS/TmpFS/Inode.h>
#include <kernel/LockGuard.h>
#include <kernel/Lock/LockGuard.h>
#include <kernel/Memory/PageTable.h>
#include <kernel/SpinLock.h>
namespace Kernel
{
@@ -119,7 +118,7 @@ namespace Kernel
private:
const dev_t m_rdev;
RecursiveSpinLock m_lock;
Mutex m_mutex;
BAN::HashMap<ino_t, BAN::RefPtr<TmpInode>> m_inode_cache;
BAN::RefPtr<TmpDirectoryInode> m_root_inode;
@@ -155,7 +154,7 @@ namespace Kernel
template<TmpFuncs::with_block_buffer_callback F>
void TmpFileSystem::with_block_buffer(size_t index, F callback)
{
LockGuard _(m_lock);
LockGuard _(m_mutex);
paddr_t block_paddr = find_block(index);
PageTable::with_fast_page(block_paddr, [&] {
BAN::ByteSpan buffer(reinterpret_cast<uint8_t*>(PageTable::fast_page()), PAGE_SIZE);
@@ -166,7 +165,7 @@ namespace Kernel
template<TmpFuncs::for_each_inode_callback F>
void TmpFileSystem::for_each_inode(F callback)
{
LockGuard _(m_lock);
LockGuard _(m_mutex);
for (auto& [_, inode] : m_inode_cache)
{
switch (callback(inode))

View File

@@ -3,7 +3,7 @@
#include <BAN/String.h>
#include <BAN/Vector.h>
#include <kernel/FS/FileSystem.h>
#include <kernel/SpinLock.h>
#include <kernel/Lock/Mutex.h>
namespace Kernel
{
@@ -42,7 +42,7 @@ namespace Kernel
MountPoint* mount_from_root_inode(BAN::RefPtr<Inode>);
private:
SpinLock m_lock;
Mutex m_mutex;
FileSystem* m_root_fs = nullptr;
BAN::Vector<MountPoint> m_mount_points;
};

View File

@@ -5,7 +5,7 @@
#include <kernel/Device/Device.h>
#include <kernel/Input/PS2/Config.h>
#include <kernel/InterruptController.h>
#include <kernel/SpinLock.h>
#include <kernel/Lock/Mutex.h>
namespace Kernel::Input
{
@@ -60,7 +60,7 @@ namespace Kernel::Input
private:
BAN::RefPtr<PS2Device> m_devices[2];
RecursiveSpinLock m_lock;
Mutex m_mutex;
BAN::CircularQueue<Command, 128> m_command_queue;
uint64_t m_command_send_time { 0 };

View File

@@ -51,6 +51,11 @@ namespace Kernel
bool m_using_apic { false };
};
bool interrupts_enabled();
inline bool interrupts_enabled()
{
uintptr_t flags;
asm volatile("pushf; pop %0" : "=r"(flags) :: "memory");
return flags & (1 << 9);
}
}

View File

@@ -0,0 +1,131 @@
#pragma once
#include <BAN/Atomic.h>
#include <BAN/NoCopyMove.h>
#include <kernel/Scheduler.h>
#include <sys/types.h>
namespace Kernel
{
class Mutex
{
BAN_NON_COPYABLE(Mutex);
BAN_NON_MOVABLE(Mutex);
public:
Mutex() = default;
void lock()
{
auto tid = Scheduler::current_tid();
if (tid == m_locker)
ASSERT_GT(m_lock_depth, 0);
else
{
while (!m_locker.compare_exchange(-1, tid))
Scheduler::get().reschedule();
ASSERT_EQ(m_lock_depth, 0);
}
m_lock_depth++;
}
bool try_lock()
{
auto tid = Scheduler::current_tid();
if (tid == m_locker)
ASSERT_GT(m_lock_depth, 0);
else
{
if (!m_locker.compare_exchange(-1, tid))
return false;
ASSERT_EQ(m_lock_depth, 0);
}
m_lock_depth++;
}
void unlock()
{
ASSERT_EQ(m_locker.load(), Scheduler::current_tid());
ASSERT_GT(m_lock_depth, 0);
if (--m_lock_depth == 0)
m_locker = -1;
}
pid_t locker() const { return m_locker; }
bool is_locked() const { return m_locker != -1; }
uint32_t lock_depth() const { return m_lock_depth; }
private:
BAN::Atomic<pid_t> m_locker { -1 };
uint32_t m_lock_depth { 0 };
};
class PriorityMutex
{
BAN_NON_COPYABLE(PriorityMutex);
BAN_NON_MOVABLE(PriorityMutex);
public:
PriorityMutex() = default;
void lock()
{
auto tid = Scheduler::current_tid();
if (tid == m_locker)
ASSERT_GT(m_lock_depth, 0);
else
{
bool has_priority = tid ? !Thread::current().is_userspace() : true;
if (has_priority)
m_queue_length++;
while (!(has_priority || m_queue_length == 0) || !m_locker.compare_exchange(-1, tid))
Scheduler::get().reschedule();
ASSERT_EQ(m_lock_depth, 0);
}
m_lock_depth++;
}
bool try_lock()
{
auto tid = Scheduler::current_tid();
if (tid == m_locker)
ASSERT_GT(m_lock_depth, 0);
else
{
bool has_priority = tid ? !Thread::current().is_userspace() : true;
if (!(has_priority || m_queue_length == 0) || !m_locker.compare_exchange(-1, tid))
return false;
if (has_priority)
m_queue_length++;
ASSERT_EQ(m_lock_depth, 0);
}
m_lock_depth++;
}
void unlock()
{
auto tid = Scheduler::current_tid();
ASSERT_EQ(m_locker.load(), tid);
ASSERT_GT(m_lock_depth, 0);
if (--m_lock_depth == 0)
{
bool has_priority = tid ? !Thread::current().is_userspace() : true;
if (has_priority)
m_queue_length--;
m_locker = -1;
}
}
pid_t locker() const { return m_locker; }
bool is_locked() const { return m_locker != -1; }
uint32_t lock_depth() const { return m_lock_depth; }
private:
BAN::Atomic<pid_t> m_locker { -1 };
uint32_t m_lock_depth { 0 };
BAN::Atomic<uint32_t> m_queue_length { 0 };
};
}

View File

@@ -0,0 +1,67 @@
#pragma once
#include <BAN/Atomic.h>
#include <BAN/NoCopyMove.h>
#include <sys/types.h>
namespace Kernel
{
using InterruptState = bool;
class SpinLock
{
BAN_NON_COPYABLE(SpinLock);
BAN_NON_MOVABLE(SpinLock);
public:
SpinLock() = default;
InterruptState lock();
void unlock(InterruptState state);
private:
BAN::Atomic<pid_t> m_locker { -1 };
};
class RecursiveSpinLock
{
BAN_NON_COPYABLE(RecursiveSpinLock);
BAN_NON_MOVABLE(RecursiveSpinLock);
public:
RecursiveSpinLock() = default;
InterruptState lock();
void unlock(InterruptState state);
private:
BAN::Atomic<pid_t> m_locker { -1 };
uint32_t m_lock_depth { 0 };
};
template<typename Lock>
class SpinLockGuard
{
BAN_NON_COPYABLE(SpinLockGuard);
BAN_NON_MOVABLE(SpinLockGuard);
public:
SpinLockGuard(Lock& lock)
: m_lock(lock)
{
m_state = m_lock.lock();
}
~SpinLockGuard()
{
m_lock.unlock(m_state);
}
private:
Lock& m_lock;
InterruptState m_state;
};
}

View File

@@ -3,8 +3,8 @@
#include <BAN/NoCopyMove.h>
#include <BAN/Vector.h>
#include <kernel/Lock/SpinLock.h>
#include <kernel/Memory/PhysicalRange.h>
#include <kernel/SpinLock.h>
namespace Kernel
{

View File

@@ -3,8 +3,8 @@
#include <BAN/Errors.h>
#include <BAN/Traits.h>
#include <kernel/CriticalScope.h>
#include <kernel/Lock/SpinLock.h>
#include <kernel/Memory/Types.h>
#include <kernel/SpinLock.h>
namespace Kernel
{
@@ -15,6 +15,12 @@ namespace Kernel
requires BAN::is_same_v<decltype(func()), void>;
};
template<typename F>
concept with_fast_page_callback_error = requires(F func)
{
requires BAN::is_same_v<decltype(func()), BAN::ErrorOr<void>>;
};
class PageTable
{
public:
@@ -37,19 +43,27 @@ namespace Kernel
static PageTable& kernel();
static PageTable& current();
static void map_fast_page(paddr_t);
static void unmap_fast_page();
static constexpr vaddr_t fast_page() { return KERNEL_OFFSET; }
template<with_fast_page_callback F>
static void with_fast_page(paddr_t paddr, F callback)
{
CriticalScope _;
SpinLockGuard _(s_fast_page_lock);
map_fast_page(paddr);
callback();
unmap_fast_page();
}
template<with_fast_page_callback_error F>
static BAN::ErrorOr<void> with_fast_page(paddr_t paddr, F callback)
{
SpinLockGuard _(s_fast_page_lock);
map_fast_page(paddr);
auto ret = callback();
unmap_fast_page();
return ret;
}
// FIXME: implement sized checks, return span, etc
static void* fast_page_as_ptr(size_t offset = 0)
{
@@ -97,8 +111,8 @@ namespace Kernel
void load();
void lock() const { m_lock.lock(); }
void unlock() const { m_lock.unlock(); }
InterruptState lock() const { return m_lock.lock(); }
void unlock(InterruptState state) const { m_lock.unlock(state); }
void debug_dump();
@@ -110,9 +124,13 @@ namespace Kernel
void prepare_fast_page();
static void invalidate(vaddr_t);
static void map_fast_page(paddr_t);
static void unmap_fast_page();
private:
paddr_t m_highest_paging_struct { 0 };
mutable RecursiveSpinLock m_lock;
static SpinLock s_fast_page_lock;
};
static constexpr size_t range_page_count(vaddr_t start, size_t bytes)

View File

@@ -1,33 +0,0 @@
#pragma once
#include <kernel/CriticalScope.h>
#include <kernel/LockGuard.h>
#include <kernel/Memory/PageTable.h>
namespace Kernel
{
class PageTableScope
{
public:
PageTableScope(PageTable& page_table)
: m_guard(page_table)
, m_old(PageTable::current())
, m_temp(page_table)
{
if (&m_old != &m_temp)
m_temp.load();
}
~PageTableScope()
{
if (&m_old != &m_temp)
m_old.load();
}
private:
LockGuard<PageTable> m_guard;
CriticalScope m_scope;
PageTable& m_old;
PageTable& m_temp;
};
}

View File

@@ -51,7 +51,7 @@ namespace Kernel
};
private:
SpinLock m_lock;
SpinLock m_table_lock;
BAN::HashMap<BAN::IPv4Address, BAN::MACAddress> m_arp_table;

View File

@@ -12,7 +12,6 @@
#include <kernel/Networking/NetworkLayer.h>
#include <kernel/Networking/NetworkSocket.h>
#include <kernel/Process.h>
#include <kernel/SpinLock.h>
namespace Kernel
{
@@ -68,7 +67,7 @@ namespace Kernel
};
private:
RecursiveSpinLock m_lock;
RecursiveSpinLock m_bound_socket_lock;
BAN::UniqPtr<ARPTable> m_arp_table;
Process* m_process { nullptr };

View File

@@ -1,6 +1,7 @@
#pragma once
#include <BAN/Endianness.h>
#include <kernel/Lock/Mutex.h>
#include <kernel/Memory/VirtualRange.h>
#include <kernel/Networking/NetworkInterface.h>
#include <kernel/Networking/NetworkSocket.h>
@@ -119,8 +120,8 @@ namespace Kernel
uint64_t m_time_wait_start_ms { 0 };
RecursiveSpinLock m_lock;
Semaphore m_semaphore;
Mutex m_lock;
Semaphore m_semaphore;
BAN::Atomic<bool> m_should_ack { false };

View File

@@ -2,6 +2,7 @@
#include <BAN/CircularQueue.h>
#include <BAN/Endianness.h>
#include <kernel/Lock/SpinLock.h>
#include <kernel/Memory/VirtualRange.h>
#include <kernel/Networking/NetworkInterface.h>
#include <kernel/Networking/NetworkSocket.h>

View File

@@ -4,6 +4,7 @@
#include <BAN/WeakPtr.h>
#include <kernel/FS/Socket.h>
#include <kernel/FS/TmpFS/Inode.h>
#include <kernel/Lock/SpinLock.h>
namespace Kernel
{
@@ -65,6 +66,7 @@ namespace Kernel
BAN::CircularQueue<size_t, 128> m_packet_sizes;
size_t m_packet_size_total { 0 };
BAN::UniqPtr<VirtualRange> m_packet_buffer;
SpinLock m_packet_lock;
Semaphore m_packet_semaphore;
friend class BAN::RefPtr<UnixDomainSocket>;

View File

@@ -7,10 +7,10 @@
#include <BAN/Vector.h>
#include <kernel/Credentials.h>
#include <kernel/FS/Inode.h>
#include <kernel/Lock/Mutex.h>
#include <kernel/Memory/Heap.h>
#include <kernel/Memory/MemoryRegion.h>
#include <kernel/OpenFileDescriptorSet.h>
#include <kernel/SpinLock.h>
#include <kernel/Terminal/TTY.h>
#include <kernel/Thread.h>
@@ -51,9 +51,6 @@ namespace Kernel
void register_to_scheduler();
void exit(int status, int signal);
static void for_each_process(const BAN::Function<BAN::Iteration(Process&)>& callback);
static void for_each_process_in_session(pid_t sid, const BAN::Function<BAN::Iteration(Process&)>& callback);
void add_thread(Thread*);
void on_thread_exit(Thread&);
@@ -222,7 +219,7 @@ namespace Kernel
const pid_t m_pid;
const pid_t m_parent;
mutable RecursiveSpinLock m_lock;
mutable Mutex m_process_lock;
BAN::String m_working_directory;
BAN::Vector<Thread*> m_threads;

View File

@@ -1,65 +0,0 @@
#pragma once
#include <BAN/Atomic.h>
#include <BAN/NoCopyMove.h>
#include <sys/types.h>
namespace Kernel
{
class SpinLock
{
BAN_NON_COPYABLE(SpinLock);
BAN_NON_MOVABLE(SpinLock);
public:
SpinLock() = default;
void lock();
void unlock();
bool is_locked() const;
uint32_t lock_depth() const { return m_locker != -1; }
private:
BAN::Atomic<pid_t> m_locker = -1;
};
class RecursiveSpinLock
{
BAN_NON_COPYABLE(RecursiveSpinLock);
BAN_NON_MOVABLE(RecursiveSpinLock);
public:
RecursiveSpinLock() = default;
void lock();
void unlock();
bool is_locked() const;
uint32_t lock_depth() const { return m_lock_depth; }
private:
BAN::Atomic<pid_t> m_locker = -1;
BAN::Atomic<uint32_t> m_lock_depth = 0;
};
class RecursivePrioritySpinLock
{
BAN_NON_COPYABLE(RecursivePrioritySpinLock);
BAN_NON_MOVABLE(RecursivePrioritySpinLock);
public:
RecursivePrioritySpinLock() = default;
void lock();
void unlock();
bool is_locked() const;
uint32_t lock_depth() const { return m_lock_depth; }
private:
BAN::Atomic<pid_t> m_locker = -1;
BAN::Atomic<uint32_t> m_lock_depth = 0;
BAN::Atomic<uint32_t> m_queue_length = 0;
};
}

View File

@@ -4,7 +4,7 @@
#include <BAN/RefPtr.h>
#include <BAN/Vector.h>
#include <kernel/InterruptController.h>
#include <kernel/SpinLock.h>
#include <kernel/Lock/Mutex.h>
namespace Kernel
{
@@ -51,7 +51,7 @@ namespace Kernel
private:
const uint16_t m_base;
const uint16_t m_ctrl;
SpinLock m_lock;
Mutex m_mutex;
volatile bool m_has_got_irq { false };

View File

@@ -3,7 +3,6 @@
#include <BAN/Array.h>
#include <BAN/ByteSpan.h>
#include <kernel/Memory/Types.h>
#include <kernel/SpinLock.h>
namespace Kernel
{

View File

@@ -3,6 +3,7 @@
#include <BAN/UniqPtr.h>
#include <BAN/Vector.h>
#include <kernel/InterruptController.h>
#include <kernel/Lock/Mutex.h>
#include <kernel/Memory/DMARegion.h>
#include <kernel/Semaphore.h>
#include <kernel/Storage/NVMe/Definitions.h>
@@ -20,7 +21,7 @@ namespace Kernel
virtual void handle_irq() final override;
private:
SpinLock m_lock;
Mutex m_mutex;
BAN::UniqPtr<Kernel::DMARegion> m_completion_queue;
BAN::UniqPtr<Kernel::DMARegion> m_submission_queue;
volatile NVMe::DoorbellRegisters& m_doorbell;

View File

@@ -2,6 +2,7 @@
#include <BAN/Vector.h>
#include <kernel/Device/Device.h>
#include <kernel/Lock/Mutex.h>
#include <kernel/Storage/DiskCache.h>
#include <kernel/Storage/Partition.h>
@@ -44,7 +45,7 @@ namespace Kernel
virtual bool has_error_impl() const override { return false; }
private:
SpinLock m_lock;
Mutex m_mutex;
BAN::Optional<DiskCache> m_disk_cache;
BAN::Vector<BAN::RefPtr<Partition>> m_partitions;

View File

@@ -3,7 +3,7 @@
#include <BAN/Array.h>
#include <kernel/Device/Device.h>
#include <kernel/Input/KeyEvent.h>
#include <kernel/SpinLock.h>
#include <kernel/Lock/SpinLock.h>
#include <kernel/Terminal/TerminalDriver.h>
#include <kernel/Terminal/termios.h>
#include <kernel/Semaphore.h>
@@ -84,6 +84,8 @@ namespace Kernel
Semaphore semaphore;
};
Buffer m_output;
RecursiveSpinLock m_write_lock;
};
}

View File

@@ -3,7 +3,6 @@
#include <BAN/Array.h>
#include <kernel/Device/Device.h>
#include <kernel/Input/KeyEvent.h>
#include <kernel/SpinLock.h>
#include <kernel/Terminal/TerminalDriver.h>
#include <kernel/Terminal/termios.h>
#include <kernel/Terminal/TTY.h>