Kernel: Rewrite all kernel mutexes

Now SpinLock is actually just a spin lock and I added a Mutex that
does the same as the old "SpinLock". This is in preparation for
starting to support smp and making the kernel smp safe. This commit
also removes obsolete PageTableScope and CriticalScope which should
now be used by alternative APIs.
This commit is contained in:
2024-02-25 21:29:43 +02:00
parent 6ebfe05fce
commit 40b626b0aa
83 changed files with 825 additions and 703 deletions

View File

@@ -2,6 +2,7 @@
#include <BAN/Vector.h>
#include <kernel/InterruptController.h>
#include <kernel/Lock/SpinLock.h>
#include <kernel/Memory/Types.h>
namespace Kernel
@@ -58,6 +59,7 @@ namespace Kernel
BAN::Vector<IOAPIC> m_io_apics;
uint8_t m_irq_overrides[0x100] {};
uint8_t m_reserved_gsis[0x100 / 8] {};
SpinLock m_lock;
};
}

View File

@@ -1,30 +0,0 @@
#pragma once
#include <BAN/NoCopyMove.h>
#include <stddef.h>
namespace Kernel
{
class CriticalScope
{
BAN_NON_COPYABLE(CriticalScope);
BAN_NON_MOVABLE(CriticalScope);
public:
CriticalScope()
{
asm volatile("pushf; cli; pop %0" : "=r"(m_flags) :: "memory");
}
~CriticalScope()
{
asm volatile("push %0; popf" :: "rm"(m_flags) : "memory", "cc");
}
private:
size_t m_flags;
};
}

View File

@@ -1,32 +1,33 @@
#pragma once
#include <BAN/Formatter.h>
#include <kernel/Lock/SpinLock.h>
#define dprintln(...) \
do { \
Debug::DebugLock::lock(); \
Debug::s_debug_lock.lock(); \
Debug::print_prefix(__FILE__, __LINE__); \
BAN::Formatter::print(Debug::putchar, __VA_ARGS__); \
BAN::Formatter::print(Debug::putchar, "\r\n"); \
Debug::DebugLock::unlock(); \
Debug::s_debug_lock.unlock(); \
} while(false)
#define dwarnln(...) \
do { \
Debug::DebugLock::lock(); \
Debug::s_debug_lock.lock(); \
BAN::Formatter::print(Debug::putchar, "\e[33m"); \
dprintln(__VA_ARGS__); \
BAN::Formatter::print(Debug::putchar, "\e[m"); \
Debug::DebugLock::unlock(); \
Debug::s_debug_lock.unlock(); \
} while(false)
#define derrorln(...) \
do { \
Debug::DebugLock::lock(); \
Debug::s_debug_lock.lock(); \
BAN::Formatter::print(Debug::putchar, "\e[31m"); \
dprintln(__VA_ARGS__); \
BAN::Formatter::print(Debug::putchar, "\e[m"); \
Debug::DebugLock::unlock(); \
Debug::s_debug_lock.unlock(); \
} while(false)
#define dprintln_if(cond, ...) \
@@ -55,10 +56,5 @@ namespace Debug
void putchar(char);
void print_prefix(const char*, int);
class DebugLock
{
public:
static void lock();
static void unlock();
};
extern Kernel::RecursiveSpinLock s_debug_lock;
}

View File

@@ -29,7 +29,7 @@ namespace Kernel
{ }
private:
mutable SpinLock m_device_lock;
mutable Mutex m_device_lock;
BAN::Vector<BAN::RefPtr<Device>> m_devices;

View File

@@ -106,7 +106,7 @@ namespace Kernel
};
private:
RecursiveSpinLock m_lock;
Mutex m_mutex;
BAN::RefPtr<BlockDevice> m_block_device;

View File

@@ -9,7 +9,7 @@
#include <kernel/API/DirectoryEntry.h>
#include <kernel/Credentials.h>
#include <kernel/SpinLock.h>
#include <kernel/Lock/Mutex.h>
#include <sys/socket.h>
#include <sys/types.h>
@@ -157,7 +157,7 @@ namespace Kernel
virtual BAN::ErrorOr<long> ioctl_impl(int request, void* arg) { return BAN::Error::from_errno(ENOTSUP); }
protected:
mutable RecursivePrioritySpinLock m_lock;
mutable Mutex m_mutex;
private:
BAN::WeakPtr<SharedFileData> m_shared_region;

View File

@@ -2,7 +2,6 @@
#include <kernel/FS/Inode.h>
#include <kernel/Semaphore.h>
#include <kernel/SpinLock.h>
namespace Kernel
{

View File

@@ -4,9 +4,8 @@
#include <BAN/Iteration.h>
#include <kernel/FS/FileSystem.h>
#include <kernel/FS/TmpFS/Inode.h>
#include <kernel/LockGuard.h>
#include <kernel/Lock/LockGuard.h>
#include <kernel/Memory/PageTable.h>
#include <kernel/SpinLock.h>
namespace Kernel
{
@@ -119,7 +118,7 @@ namespace Kernel
private:
const dev_t m_rdev;
RecursiveSpinLock m_lock;
Mutex m_mutex;
BAN::HashMap<ino_t, BAN::RefPtr<TmpInode>> m_inode_cache;
BAN::RefPtr<TmpDirectoryInode> m_root_inode;
@@ -155,7 +154,7 @@ namespace Kernel
template<TmpFuncs::with_block_buffer_callback F>
void TmpFileSystem::with_block_buffer(size_t index, F callback)
{
LockGuard _(m_lock);
LockGuard _(m_mutex);
paddr_t block_paddr = find_block(index);
PageTable::with_fast_page(block_paddr, [&] {
BAN::ByteSpan buffer(reinterpret_cast<uint8_t*>(PageTable::fast_page()), PAGE_SIZE);
@@ -166,7 +165,7 @@ namespace Kernel
template<TmpFuncs::for_each_inode_callback F>
void TmpFileSystem::for_each_inode(F callback)
{
LockGuard _(m_lock);
LockGuard _(m_mutex);
for (auto& [_, inode] : m_inode_cache)
{
switch (callback(inode))

View File

@@ -3,7 +3,7 @@
#include <BAN/String.h>
#include <BAN/Vector.h>
#include <kernel/FS/FileSystem.h>
#include <kernel/SpinLock.h>
#include <kernel/Lock/Mutex.h>
namespace Kernel
{
@@ -42,7 +42,7 @@ namespace Kernel
MountPoint* mount_from_root_inode(BAN::RefPtr<Inode>);
private:
SpinLock m_lock;
Mutex m_mutex;
FileSystem* m_root_fs = nullptr;
BAN::Vector<MountPoint> m_mount_points;
};

View File

@@ -20,6 +20,8 @@ namespace Kernel::Input
KeyboardLayout();
private:
SpinLock m_lock;
BAN::Array<Key, 0xFF> m_keycode_to_key_normal;
BAN::Array<Key, 0xFF> m_keycode_to_key_shift;
BAN::Array<Key, 0xFF> m_keycode_to_key_altgr;

View File

@@ -5,7 +5,6 @@
#include <kernel/Device/Device.h>
#include <kernel/Input/PS2/Config.h>
#include <kernel/InterruptController.h>
#include <kernel/SpinLock.h>
namespace Kernel::Input
{
@@ -60,7 +59,8 @@ namespace Kernel::Input
private:
BAN::RefPtr<PS2Device> m_devices[2];
RecursiveSpinLock m_lock;
Mutex m_mutex;
RecursiveSpinLock m_cmd_lock;
BAN::CircularQueue<Command, 128> m_command_queue;
uint64_t m_command_send_time { 0 };

View File

@@ -45,6 +45,7 @@ namespace Kernel::Input
PS2Keymap m_keymap;
Semaphore m_semaphore;
SpinLock m_event_lock;
protected:
virtual BAN::ErrorOr<size_t> read_impl(off_t, BAN::ByteSpan) override;

View File

@@ -38,6 +38,7 @@ namespace Kernel::Input
BAN::CircularQueue<MouseEvent, 128> m_event_queue;
SpinLock m_event_lock;
Semaphore m_semaphore;
protected:

View File

@@ -0,0 +1,54 @@
#pragma once
#include <BAN/Atomic.h>
#include <BAN/NoCopyMove.h>
#include <sys/types.h>
namespace Kernel
{
class Mutex
{
BAN_NON_COPYABLE(Mutex);
BAN_NON_MOVABLE(Mutex);
public:
Mutex() = default;
void lock();
bool try_lock();
void unlock();
pid_t locker() const { return m_locker; }
bool is_locked() const { return m_locker != -1; }
uint32_t lock_depth() const { return m_lock_depth; }
private:
BAN::Atomic<pid_t> m_locker { -1 };
uint32_t m_lock_depth { 0 };
};
class PriorityMutex
{
BAN_NON_COPYABLE(PriorityMutex);
BAN_NON_MOVABLE(PriorityMutex);
public:
PriorityMutex() = default;
void lock();
bool try_lock();
void unlock();
pid_t locker() const { return m_locker; }
bool is_locked() const { return m_locker != -1; }
uint32_t lock_depth() const { return m_lock_depth; }
private:
BAN::Atomic<pid_t> m_locker { -1 };
uint32_t m_lock_depth { 0 };
BAN::Atomic<uint32_t> m_queue_depth { 0 };
};
}

View File

@@ -0,0 +1,54 @@
#pragma once
#include <BAN/Atomic.h>
#include <BAN/NoCopyMove.h>
#include <sys/types.h>
namespace Kernel
{
class SpinLock
{
BAN_NON_COPYABLE(SpinLock);
BAN_NON_MOVABLE(SpinLock);
public:
SpinLock() = default;
void lock();
bool try_lock();
void unlock();
pid_t locker() const { return m_locker; }
bool is_locked() const { return m_locker != -1; }
uint32_t lock_depth() const { return is_locked(); }
private:
BAN::Atomic<pid_t> m_locker { -1 };
uintptr_t m_flags { 0 };
};
class RecursiveSpinLock
{
BAN_NON_COPYABLE(RecursiveSpinLock);
BAN_NON_MOVABLE(RecursiveSpinLock);
public:
RecursiveSpinLock() = default;
void lock();
bool try_lock();
void unlock();
pid_t locker() const { return m_locker; }
bool is_locked() const { return m_locker != -1; }
uint32_t lock_depth() const { return m_lock_depth; }
private:
BAN::Atomic<pid_t> m_locker { -1 };
uint32_t m_lock_depth { 0 };
uintptr_t m_flags { 0 };
};
}

View File

@@ -3,8 +3,8 @@
#include <BAN/NoCopyMove.h>
#include <BAN/Vector.h>
#include <kernel/Lock/SpinLock.h>
#include <kernel/Memory/PhysicalRange.h>
#include <kernel/SpinLock.h>
namespace Kernel
{

View File

@@ -2,9 +2,8 @@
#include <BAN/Errors.h>
#include <BAN/Traits.h>
#include <kernel/CriticalScope.h>
#include <kernel/Lock/SpinLock.h>
#include <kernel/Memory/Types.h>
#include <kernel/SpinLock.h>
namespace Kernel
{
@@ -15,6 +14,12 @@ namespace Kernel
requires BAN::is_same_v<decltype(func()), void>;
};
template<typename F>
concept with_fast_page_callback_error = requires(F func)
{
requires BAN::is_same_v<decltype(func()), BAN::ErrorOr<void>>;
};
class PageTable
{
public:
@@ -37,19 +42,30 @@ namespace Kernel
static PageTable& kernel();
static PageTable& current();
static void map_fast_page(paddr_t);
static void unmap_fast_page();
static constexpr vaddr_t fast_page() { return KERNEL_OFFSET; }
public:
template<with_fast_page_callback F>
static void with_fast_page(paddr_t paddr, F callback)
{
CriticalScope _;
s_fast_page_lock.lock();
map_fast_page(paddr);
callback();
unmap_fast_page();
s_fast_page_lock.unlock();
}
template<with_fast_page_callback_error F>
static BAN::ErrorOr<void> with_fast_page(paddr_t paddr, F callback)
{
s_fast_page_lock.lock();
map_fast_page(paddr);
auto ret = callback();
unmap_fast_page();
s_fast_page_lock.unlock();
return ret;
}
static constexpr vaddr_t fast_page() { return KERNEL_OFFSET; }
// FIXME: implement sized checks, return span, etc
static void* fast_page_as_ptr(size_t offset = 0)
{
@@ -110,9 +126,13 @@ namespace Kernel
void prepare_fast_page();
static void invalidate(vaddr_t);
static void map_fast_page(paddr_t);
static void unmap_fast_page();
private:
paddr_t m_highest_paging_struct { 0 };
mutable RecursiveSpinLock m_lock;
static SpinLock s_fast_page_lock;
};
static constexpr size_t range_page_count(vaddr_t start, size_t bytes)

View File

@@ -1,33 +0,0 @@
#pragma once
#include <kernel/CriticalScope.h>
#include <kernel/LockGuard.h>
#include <kernel/Memory/PageTable.h>
namespace Kernel
{
class PageTableScope
{
public:
PageTableScope(PageTable& page_table)
: m_guard(page_table)
, m_old(PageTable::current())
, m_temp(page_table)
{
if (&m_old != &m_temp)
m_temp.load();
}
~PageTableScope()
{
if (&m_old != &m_temp)
m_old.load();
}
private:
LockGuard<PageTable> m_guard;
CriticalScope m_scope;
PageTable& m_old;
PageTable& m_temp;
};
}

View File

@@ -51,7 +51,8 @@ namespace Kernel
};
private:
SpinLock m_lock;
SpinLock m_pending_lock;
SpinLock m_table_lock;
BAN::HashMap<BAN::IPv4Address, BAN::MACAddress> m_arp_table;

View File

@@ -67,6 +67,8 @@ namespace Kernel
bool m_has_eerprom { false };
private:
SpinLock m_lock;
BAN::UniqPtr<DMARegion> m_rx_buffer_region;
BAN::UniqPtr<DMARegion> m_tx_buffer_region;
BAN::UniqPtr<DMARegion> m_rx_descriptor_region;

View File

@@ -12,7 +12,6 @@
#include <kernel/Networking/NetworkLayer.h>
#include <kernel/Networking/NetworkSocket.h>
#include <kernel/Process.h>
#include <kernel/SpinLock.h>
namespace Kernel
{
@@ -68,7 +67,8 @@ namespace Kernel
};
private:
RecursiveSpinLock m_lock;
RecursiveSpinLock m_packet_lock;
RecursiveSpinLock m_socket_lock;
BAN::UniqPtr<ARPTable> m_arp_table;
Process* m_process { nullptr };

View File

@@ -119,8 +119,7 @@ namespace Kernel
uint64_t m_time_wait_start_ms { 0 };
RecursiveSpinLock m_lock;
Semaphore m_semaphore;
Semaphore m_semaphore;
BAN::Atomic<bool> m_should_ack { false };

View File

@@ -20,8 +20,11 @@ namespace Kernel
private:
static PIC* create();
friend class InterruptController;
private:
SpinLock m_lock;
uint16_t m_reserved_irqs { 0 };
friend class InterruptController;
};
}

View File

@@ -10,7 +10,6 @@
#include <kernel/Memory/Heap.h>
#include <kernel/Memory/MemoryRegion.h>
#include <kernel/OpenFileDescriptorSet.h>
#include <kernel/SpinLock.h>
#include <kernel/Terminal/TTY.h>
#include <kernel/Thread.h>
@@ -222,7 +221,8 @@ namespace Kernel
const pid_t m_pid;
const pid_t m_parent;
mutable RecursiveSpinLock m_lock;
mutable Mutex m_big_mutex;
SpinLock m_signal_lock;
BAN::String m_working_directory;
BAN::Vector<Thread*> m_threads;

View File

@@ -7,6 +7,21 @@
namespace Kernel
{
class SchedulerLock
{
public:
void lock();
void unlock();
void unlock_all();
pid_t locker() const;
private:
BAN::Atomic<pid_t> m_locker { -1 };
uint32_t m_lock_depth { 0 };
friend class Scheduler;
};
class Scheduler
{
public:
@@ -19,6 +34,8 @@ namespace Kernel
void reschedule();
void reschedule_if_idling();
void reschedule_current_no_save();
void set_current_thread_sleeping(uint64_t wake_time);
void block_current_thread(Semaphore*, uint64_t wake_time);
@@ -29,8 +46,8 @@ namespace Kernel
Thread& current_thread();
static pid_t current_tid();
[[noreturn]] void execute_current_thread();
[[noreturn]] void _execute_current_thread();
BAN::ErrorOr<void> add_thread(Thread*);
[[noreturn]] void delete_current_process_and_thread();
private:
@@ -43,7 +60,8 @@ namespace Kernel
void remove_and_advance_current_thread();
void advance_current_thread();
BAN::ErrorOr<void> add_thread(Thread*);
[[noreturn]] void execute_current_thread();
[[noreturn]] void _execute_current_thread();
private:
struct SchedulerThread
@@ -57,13 +75,13 @@ namespace Kernel
Semaphore* semaphore;
};
SchedulerLock m_lock;
Thread* m_idle_thread { nullptr };
BAN::LinkedList<SchedulerThread> m_active_threads;
BAN::LinkedList<SchedulerThread> m_sleeping_threads;
BAN::LinkedList<SchedulerThread>::iterator m_current_thread;
friend class Process;
};
}

View File

@@ -1,65 +0,0 @@
#pragma once
#include <BAN/Atomic.h>
#include <BAN/NoCopyMove.h>
#include <sys/types.h>
namespace Kernel
{
class SpinLock
{
BAN_NON_COPYABLE(SpinLock);
BAN_NON_MOVABLE(SpinLock);
public:
SpinLock() = default;
void lock();
void unlock();
bool is_locked() const;
uint32_t lock_depth() const { return m_locker != -1; }
private:
BAN::Atomic<pid_t> m_locker = -1;
};
class RecursiveSpinLock
{
BAN_NON_COPYABLE(RecursiveSpinLock);
BAN_NON_MOVABLE(RecursiveSpinLock);
public:
RecursiveSpinLock() = default;
void lock();
void unlock();
bool is_locked() const;
uint32_t lock_depth() const { return m_lock_depth; }
private:
BAN::Atomic<pid_t> m_locker = -1;
BAN::Atomic<uint32_t> m_lock_depth = 0;
};
class RecursivePrioritySpinLock
{
BAN_NON_COPYABLE(RecursivePrioritySpinLock);
BAN_NON_MOVABLE(RecursivePrioritySpinLock);
public:
RecursivePrioritySpinLock() = default;
void lock();
void unlock();
bool is_locked() const;
uint32_t lock_depth() const { return m_lock_depth; }
private:
BAN::Atomic<pid_t> m_locker = -1;
BAN::Atomic<uint32_t> m_lock_depth = 0;
BAN::Atomic<uint32_t> m_queue_length = 0;
};
}

View File

@@ -4,7 +4,7 @@
#include <BAN/RefPtr.h>
#include <BAN/Vector.h>
#include <kernel/InterruptController.h>
#include <kernel/SpinLock.h>
#include <kernel/Lock/Mutex.h>
namespace Kernel
{
@@ -51,7 +51,7 @@ namespace Kernel
private:
const uint16_t m_base;
const uint16_t m_ctrl;
SpinLock m_lock;
Mutex m_mutex;
volatile bool m_has_got_irq { false };

View File

@@ -3,7 +3,6 @@
#include <BAN/Array.h>
#include <BAN/ByteSpan.h>
#include <kernel/Memory/Types.h>
#include <kernel/SpinLock.h>
namespace Kernel
{

View File

@@ -3,6 +3,7 @@
#include <BAN/UniqPtr.h>
#include <BAN/Vector.h>
#include <kernel/InterruptController.h>
#include <kernel/Lock/Mutex.h>
#include <kernel/Memory/DMARegion.h>
#include <kernel/Semaphore.h>
#include <kernel/Storage/NVMe/Definitions.h>
@@ -20,7 +21,7 @@ namespace Kernel
virtual void handle_irq() final override;
private:
SpinLock m_lock;
Mutex m_mutex;
BAN::UniqPtr<Kernel::DMARegion> m_completion_queue;
BAN::UniqPtr<Kernel::DMARegion> m_submission_queue;
volatile NVMe::DoorbellRegisters& m_doorbell;

View File

@@ -44,7 +44,7 @@ namespace Kernel
virtual bool has_error_impl() const override { return false; }
private:
SpinLock m_lock;
Mutex m_mutex;
BAN::Optional<DiskCache> m_disk_cache;
BAN::Vector<BAN::RefPtr<Partition>> m_partitions;

View File

@@ -59,6 +59,7 @@ namespace Kernel
bool initialize();
private:
SpinLock m_lock;
BAN::String m_name;
Serial m_serial;
BAN::CircularQueue<uint8_t, 128> m_input;

View File

@@ -3,7 +3,6 @@
#include <BAN/Array.h>
#include <kernel/Device/Device.h>
#include <kernel/Input/KeyEvent.h>
#include <kernel/SpinLock.h>
#include <kernel/Terminal/TerminalDriver.h>
#include <kernel/Terminal/termios.h>
#include <kernel/Semaphore.h>

View File

@@ -3,7 +3,6 @@
#include <BAN/Array.h>
#include <kernel/Device/Device.h>
#include <kernel/Input/KeyEvent.h>
#include <kernel/SpinLock.h>
#include <kernel/Terminal/TerminalDriver.h>
#include <kernel/Terminal/termios.h>
#include <kernel/Terminal/TTY.h>
@@ -70,6 +69,8 @@ namespace Kernel
private:
BAN::String m_name;
RecursiveSpinLock m_write_lock;
State m_state { State::Normal };
AnsiState m_ansi_state { };
UTF8State m_utf8_state { };

View File

@@ -113,6 +113,8 @@ namespace Kernel
Process* m_process { nullptr };
bool m_is_userspace { false };
mutable RecursiveSpinLock m_lock;
uintptr_t* m_return_rsp { nullptr };
uintptr_t* m_return_rip { nullptr };

View File

@@ -30,6 +30,8 @@ namespace Kernel
uint64_t read_main_counter() const;
private:
mutable SpinLock m_lock;
bool m_is_64bit { false };
uint64_t m_last_ticks { 0 };