From 40b626b0aaa8a4d24dea25c1d05bb167d924e0bd Mon Sep 17 00:00:00 2001 From: Bananymous Date: Sun, 25 Feb 2024 21:29:43 +0200 Subject: [PATCH] Kernel: Rewrite all kernel mutexes Now SpinLock is actually just a spin lock and I added a Mutex that does the same as the old "SpinLock". This is in preparation for starting to support smp and making the kernel smp safe. This commit also removes obsolete PageTableScope and CriticalScope which should now be used by alternative APIs. --- LibELF/LibELF/ELF.cpp | 3 +- LibELF/LibELF/LoadableELF.cpp | 10 +- kernel/CMakeLists.txt | 3 +- kernel/arch/x86_64/PageTable.cpp | 4 +- kernel/icxxabi.cpp | 6 - kernel/include/kernel/APIC.h | 2 + kernel/include/kernel/CriticalScope.h | 30 ---- kernel/include/kernel/Debug.h | 20 +-- kernel/include/kernel/FS/DevFS/FileSystem.h | 2 +- kernel/include/kernel/FS/Ext2/FileSystem.h | 2 +- kernel/include/kernel/FS/Inode.h | 4 +- kernel/include/kernel/FS/Pipe.h | 1 - kernel/include/kernel/FS/TmpFS/FileSystem.h | 9 +- kernel/include/kernel/FS/VirtualFileSystem.h | 4 +- kernel/include/kernel/Input/KeyboardLayout.h | 2 + kernel/include/kernel/Input/PS2/Controller.h | 4 +- kernel/include/kernel/Input/PS2/Keyboard.h | 1 + kernel/include/kernel/Input/PS2/Mouse.h | 1 + kernel/include/kernel/{ => Lock}/LockGuard.h | 0 kernel/include/kernel/Lock/Mutex.h | 54 ++++++ kernel/include/kernel/Lock/SpinLock.h | 54 ++++++ kernel/include/kernel/Memory/Heap.h | 2 +- kernel/include/kernel/Memory/PageTable.h | 34 +++- kernel/include/kernel/Memory/PageTableScope.h | 33 ---- kernel/include/kernel/Networking/ARPTable.h | 3 +- .../include/kernel/Networking/E1000/E1000.h | 2 + kernel/include/kernel/Networking/IPv4Layer.h | 4 +- kernel/include/kernel/Networking/TCPSocket.h | 3 +- kernel/include/kernel/PIC.h | 5 +- kernel/include/kernel/Process.h | 4 +- kernel/include/kernel/Scheduler.h | 28 ++- kernel/include/kernel/SpinLock.h | 65 ------- kernel/include/kernel/Storage/ATA/ATABus.h | 4 +- kernel/include/kernel/Storage/DiskCache.h | 1 - kernel/include/kernel/Storage/NVMe/Queue.h | 3 +- kernel/include/kernel/Storage/StorageDevice.h | 2 +- kernel/include/kernel/Terminal/Serial.h | 1 + kernel/include/kernel/Terminal/TTY.h | 1 - kernel/include/kernel/Terminal/VirtualTTY.h | 3 +- kernel/include/kernel/Thread.h | 2 + kernel/include/kernel/Timer/HPET.h | 2 + kernel/kernel/ACPI.cpp | 57 +++--- kernel/kernel/APIC.cpp | 9 +- kernel/kernel/Debug.cpp | 17 +- kernel/kernel/Device/DebugDevice.cpp | 4 +- kernel/kernel/FS/DevFS/FileSystem.cpp | 2 +- kernel/kernel/FS/Ext2/FileSystem.cpp | 22 +-- kernel/kernel/FS/Inode.cpp | 46 ++--- kernel/kernel/FS/Pipe.cpp | 11 +- kernel/kernel/FS/ProcFS/FileSystem.cpp | 1 - kernel/kernel/FS/TmpFS/FileSystem.cpp | 28 +-- kernel/kernel/FS/VirtualFileSystem.cpp | 13 +- kernel/kernel/Input/KeyboardLayout.cpp | 5 +- kernel/kernel/Input/PS2/Controller.cpp | 23 +-- kernel/kernel/Input/PS2/Keyboard.cpp | 4 +- kernel/kernel/Input/PS2/Mouse.cpp | 4 +- kernel/kernel/Lock/Mutex.cpp | 68 ++++++++ kernel/kernel/Lock/SpinLock.cpp | 80 +++++++++ kernel/kernel/Lock/spin.o | Bin 0 -> 9896 bytes kernel/kernel/Memory/FileBackedRegion.cpp | 23 +-- kernel/kernel/Memory/Heap.cpp | 2 +- kernel/kernel/Memory/MemoryBackedRegion.cpp | 20 +-- kernel/kernel/Memory/VirtualRange.cpp | 24 +-- kernel/kernel/Memory/kmalloc.cpp | 28 ++- kernel/kernel/Networking/ARPTable.cpp | 12 +- kernel/kernel/Networking/E1000/E1000.cpp | 2 +- kernel/kernel/Networking/IPv4Layer.cpp | 28 +-- kernel/kernel/Networking/TCPSocket.cpp | 33 ++-- kernel/kernel/Networking/UDPSocket.cpp | 2 +- kernel/kernel/Networking/UNIX/Socket.cpp | 6 +- kernel/kernel/PIC.cpp | 8 +- kernel/kernel/Process.cpp | 165 +++++++++--------- kernel/kernel/Scheduler.cpp | 81 +++++++-- kernel/kernel/SpinLock.cpp | 86 --------- kernel/kernel/Storage/ATA/ATABus.cpp | 6 +- kernel/kernel/Storage/DiskCache.cpp | 39 ++--- kernel/kernel/Storage/NVMe/Queue.cpp | 4 +- kernel/kernel/Storage/StorageDevice.cpp | 28 ++- kernel/kernel/Terminal/Serial.cpp | 3 +- kernel/kernel/Terminal/TTY.cpp | 24 +-- kernel/kernel/Terminal/VirtualTTY.cpp | 6 +- kernel/kernel/Thread.cpp | 59 ++++--- kernel/kernel/Timer/HPET.cpp | 27 +-- 83 files changed, 825 insertions(+), 703 deletions(-) delete mode 100644 kernel/include/kernel/CriticalScope.h rename kernel/include/kernel/{ => Lock}/LockGuard.h (100%) create mode 100644 kernel/include/kernel/Lock/Mutex.h create mode 100644 kernel/include/kernel/Lock/SpinLock.h delete mode 100644 kernel/include/kernel/Memory/PageTableScope.h delete mode 100644 kernel/include/kernel/SpinLock.h create mode 100644 kernel/kernel/Lock/Mutex.cpp create mode 100644 kernel/kernel/Lock/SpinLock.cpp create mode 100644 kernel/kernel/Lock/spin.o delete mode 100644 kernel/kernel/SpinLock.cpp diff --git a/LibELF/LibELF/ELF.cpp b/LibELF/LibELF/ELF.cpp index f1b4ebe4..1929bf92 100644 --- a/LibELF/LibELF/ELF.cpp +++ b/LibELF/LibELF/ELF.cpp @@ -4,7 +4,6 @@ #ifdef __is_kernel #include -#include #include #endif @@ -26,7 +25,7 @@ namespace LibELF BAN::Vector buffer; TRY(buffer.resize(inode->size())); - TRY(inode->read(0, buffer.data(), inode->size())); + TRY(inode->read(0, { buffer.data(), inode->size() })); ELF* elf_ptr = new ELF(BAN::move(buffer)); if (elf_ptr == nullptr) diff --git a/LibELF/LibELF/LoadableELF.cpp b/LibELF/LibELF/LoadableELF.cpp index b8bccc17..16e3dc96 100644 --- a/LibELF/LibELF/LoadableELF.cpp +++ b/LibELF/LibELF/LoadableELF.cpp @@ -1,7 +1,6 @@ #include -#include #include -#include +#include #include #include @@ -315,12 +314,9 @@ namespace LibELF if (paddr == 0) return BAN::Error::from_errno(ENOMEM); - { - CriticalScope _; - PageTable::map_fast_page(paddr); + PageTable::with_fast_page(paddr, [&] { memcpy(PageTable::fast_page_as_ptr(), (void*)(start + i * PAGE_SIZE), PAGE_SIZE); - PageTable::unmap_fast_page(); - } + }); new_page_table.map_page_at(paddr, start + i * PAGE_SIZE, flags); elf->m_physical_page_count++; diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt index 30d904ad..51aaceb2 100644 --- a/kernel/CMakeLists.txt +++ b/kernel/CMakeLists.txt @@ -68,7 +68,8 @@ set(KERNEL_SOURCES kernel/Random.cpp kernel/Scheduler.cpp kernel/Semaphore.cpp - kernel/SpinLock.cpp + kernel/Lock/Mutex.cpp + kernel/Lock/SpinLock.cpp kernel/SSP.cpp kernel/Storage/ATA/AHCI/Controller.cpp kernel/Storage/ATA/AHCI/Device.cpp diff --git a/kernel/arch/x86_64/PageTable.cpp b/kernel/arch/x86_64/PageTable.cpp index f46c9078..98d3d50a 100644 --- a/kernel/arch/x86_64/PageTable.cpp +++ b/kernel/arch/x86_64/PageTable.cpp @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include #include @@ -22,6 +22,8 @@ namespace Kernel static bool s_has_nxe = false; static bool s_has_pge = false; + SpinLock PageTable::s_fast_page_lock; + // PML4 entry for kernel memory static paddr_t s_global_pml4e = 0; diff --git a/kernel/icxxabi.cpp b/kernel/icxxabi.cpp index 211b64a5..9b7c8aa9 100644 --- a/kernel/icxxabi.cpp +++ b/kernel/icxxabi.cpp @@ -1,6 +1,4 @@ -#include #include -#include #define ATEXIT_MAX_FUNCS 128 @@ -115,21 +113,17 @@ void __cxa_finalize(void *f) namespace __cxxabiv1 { - /* guard variables */ - static Kernel::SpinLock s_spin_lock; /* The ABI requires a 64-bit type. */ __extension__ typedef int __guard __attribute__((mode(__DI__))); int __cxa_guard_acquire (__guard* g) { - Kernel::LockGuard lock_guard(s_spin_lock); return !*(int*)g; } void __cxa_guard_release (__guard* g) { - Kernel::LockGuard lock_guard(s_spin_lock); *(int*)g = 1; } diff --git a/kernel/include/kernel/APIC.h b/kernel/include/kernel/APIC.h index 3553dbfc..74f87939 100644 --- a/kernel/include/kernel/APIC.h +++ b/kernel/include/kernel/APIC.h @@ -2,6 +2,7 @@ #include #include +#include #include namespace Kernel @@ -58,6 +59,7 @@ namespace Kernel BAN::Vector m_io_apics; uint8_t m_irq_overrides[0x100] {}; uint8_t m_reserved_gsis[0x100 / 8] {}; + SpinLock m_lock; }; } diff --git a/kernel/include/kernel/CriticalScope.h b/kernel/include/kernel/CriticalScope.h deleted file mode 100644 index 33b7baf0..00000000 --- a/kernel/include/kernel/CriticalScope.h +++ /dev/null @@ -1,30 +0,0 @@ -#pragma once - -#include - -#include - -namespace Kernel -{ - - class CriticalScope - { - BAN_NON_COPYABLE(CriticalScope); - BAN_NON_MOVABLE(CriticalScope); - - public: - CriticalScope() - { - asm volatile("pushf; cli; pop %0" : "=r"(m_flags) :: "memory"); - } - - ~CriticalScope() - { - asm volatile("push %0; popf" :: "rm"(m_flags) : "memory", "cc"); - } - - private: - size_t m_flags; - }; - -} diff --git a/kernel/include/kernel/Debug.h b/kernel/include/kernel/Debug.h index 1e4ae987..d16ca2ec 100644 --- a/kernel/include/kernel/Debug.h +++ b/kernel/include/kernel/Debug.h @@ -1,32 +1,33 @@ #pragma once #include +#include #define dprintln(...) \ do { \ - Debug::DebugLock::lock(); \ + Debug::s_debug_lock.lock(); \ Debug::print_prefix(__FILE__, __LINE__); \ BAN::Formatter::print(Debug::putchar, __VA_ARGS__); \ BAN::Formatter::print(Debug::putchar, "\r\n"); \ - Debug::DebugLock::unlock(); \ + Debug::s_debug_lock.unlock(); \ } while(false) #define dwarnln(...) \ do { \ - Debug::DebugLock::lock(); \ + Debug::s_debug_lock.lock(); \ BAN::Formatter::print(Debug::putchar, "\e[33m"); \ dprintln(__VA_ARGS__); \ BAN::Formatter::print(Debug::putchar, "\e[m"); \ - Debug::DebugLock::unlock(); \ + Debug::s_debug_lock.unlock(); \ } while(false) #define derrorln(...) \ do { \ - Debug::DebugLock::lock(); \ + Debug::s_debug_lock.lock(); \ BAN::Formatter::print(Debug::putchar, "\e[31m"); \ dprintln(__VA_ARGS__); \ BAN::Formatter::print(Debug::putchar, "\e[m"); \ - Debug::DebugLock::unlock(); \ + Debug::s_debug_lock.unlock(); \ } while(false) #define dprintln_if(cond, ...) \ @@ -55,10 +56,5 @@ namespace Debug void putchar(char); void print_prefix(const char*, int); - class DebugLock - { - public: - static void lock(); - static void unlock(); - }; + extern Kernel::RecursiveSpinLock s_debug_lock; } diff --git a/kernel/include/kernel/FS/DevFS/FileSystem.h b/kernel/include/kernel/FS/DevFS/FileSystem.h index 333201e7..f45847ff 100644 --- a/kernel/include/kernel/FS/DevFS/FileSystem.h +++ b/kernel/include/kernel/FS/DevFS/FileSystem.h @@ -29,7 +29,7 @@ namespace Kernel { } private: - mutable SpinLock m_device_lock; + mutable Mutex m_device_lock; BAN::Vector> m_devices; diff --git a/kernel/include/kernel/FS/Ext2/FileSystem.h b/kernel/include/kernel/FS/Ext2/FileSystem.h index 95ab1978..8949aaf3 100644 --- a/kernel/include/kernel/FS/Ext2/FileSystem.h +++ b/kernel/include/kernel/FS/Ext2/FileSystem.h @@ -106,7 +106,7 @@ namespace Kernel }; private: - RecursiveSpinLock m_lock; + Mutex m_mutex; BAN::RefPtr m_block_device; diff --git a/kernel/include/kernel/FS/Inode.h b/kernel/include/kernel/FS/Inode.h index d5954d20..4d5bc90b 100644 --- a/kernel/include/kernel/FS/Inode.h +++ b/kernel/include/kernel/FS/Inode.h @@ -9,7 +9,7 @@ #include #include -#include +#include #include #include @@ -157,7 +157,7 @@ namespace Kernel virtual BAN::ErrorOr ioctl_impl(int request, void* arg) { return BAN::Error::from_errno(ENOTSUP); } protected: - mutable RecursivePrioritySpinLock m_lock; + mutable Mutex m_mutex; private: BAN::WeakPtr m_shared_region; diff --git a/kernel/include/kernel/FS/Pipe.h b/kernel/include/kernel/FS/Pipe.h index ead80a3f..2d36dd72 100644 --- a/kernel/include/kernel/FS/Pipe.h +++ b/kernel/include/kernel/FS/Pipe.h @@ -2,7 +2,6 @@ #include #include -#include namespace Kernel { diff --git a/kernel/include/kernel/FS/TmpFS/FileSystem.h b/kernel/include/kernel/FS/TmpFS/FileSystem.h index 3a33dac8..4740d49b 100644 --- a/kernel/include/kernel/FS/TmpFS/FileSystem.h +++ b/kernel/include/kernel/FS/TmpFS/FileSystem.h @@ -4,9 +4,8 @@ #include #include #include -#include +#include #include -#include namespace Kernel { @@ -119,7 +118,7 @@ namespace Kernel private: const dev_t m_rdev; - RecursiveSpinLock m_lock; + Mutex m_mutex; BAN::HashMap> m_inode_cache; BAN::RefPtr m_root_inode; @@ -155,7 +154,7 @@ namespace Kernel template void TmpFileSystem::with_block_buffer(size_t index, F callback) { - LockGuard _(m_lock); + LockGuard _(m_mutex); paddr_t block_paddr = find_block(index); PageTable::with_fast_page(block_paddr, [&] { BAN::ByteSpan buffer(reinterpret_cast(PageTable::fast_page()), PAGE_SIZE); @@ -166,7 +165,7 @@ namespace Kernel template void TmpFileSystem::for_each_inode(F callback) { - LockGuard _(m_lock); + LockGuard _(m_mutex); for (auto& [_, inode] : m_inode_cache) { switch (callback(inode)) diff --git a/kernel/include/kernel/FS/VirtualFileSystem.h b/kernel/include/kernel/FS/VirtualFileSystem.h index 803caed9..1b56bf32 100644 --- a/kernel/include/kernel/FS/VirtualFileSystem.h +++ b/kernel/include/kernel/FS/VirtualFileSystem.h @@ -3,7 +3,7 @@ #include #include #include -#include +#include namespace Kernel { @@ -42,7 +42,7 @@ namespace Kernel MountPoint* mount_from_root_inode(BAN::RefPtr); private: - SpinLock m_lock; + Mutex m_mutex; FileSystem* m_root_fs = nullptr; BAN::Vector m_mount_points; }; diff --git a/kernel/include/kernel/Input/KeyboardLayout.h b/kernel/include/kernel/Input/KeyboardLayout.h index 62fb7a96..761d1c01 100644 --- a/kernel/include/kernel/Input/KeyboardLayout.h +++ b/kernel/include/kernel/Input/KeyboardLayout.h @@ -20,6 +20,8 @@ namespace Kernel::Input KeyboardLayout(); private: + SpinLock m_lock; + BAN::Array m_keycode_to_key_normal; BAN::Array m_keycode_to_key_shift; BAN::Array m_keycode_to_key_altgr; diff --git a/kernel/include/kernel/Input/PS2/Controller.h b/kernel/include/kernel/Input/PS2/Controller.h index 16332da4..f4c2ef6d 100644 --- a/kernel/include/kernel/Input/PS2/Controller.h +++ b/kernel/include/kernel/Input/PS2/Controller.h @@ -5,7 +5,6 @@ #include #include #include -#include namespace Kernel::Input { @@ -60,7 +59,8 @@ namespace Kernel::Input private: BAN::RefPtr m_devices[2]; - RecursiveSpinLock m_lock; + Mutex m_mutex; + RecursiveSpinLock m_cmd_lock; BAN::CircularQueue m_command_queue; uint64_t m_command_send_time { 0 }; diff --git a/kernel/include/kernel/Input/PS2/Keyboard.h b/kernel/include/kernel/Input/PS2/Keyboard.h index 2fcd915f..d8e820d9 100644 --- a/kernel/include/kernel/Input/PS2/Keyboard.h +++ b/kernel/include/kernel/Input/PS2/Keyboard.h @@ -45,6 +45,7 @@ namespace Kernel::Input PS2Keymap m_keymap; Semaphore m_semaphore; + SpinLock m_event_lock; protected: virtual BAN::ErrorOr read_impl(off_t, BAN::ByteSpan) override; diff --git a/kernel/include/kernel/Input/PS2/Mouse.h b/kernel/include/kernel/Input/PS2/Mouse.h index f0977a2a..b3e5db06 100644 --- a/kernel/include/kernel/Input/PS2/Mouse.h +++ b/kernel/include/kernel/Input/PS2/Mouse.h @@ -38,6 +38,7 @@ namespace Kernel::Input BAN::CircularQueue m_event_queue; + SpinLock m_event_lock; Semaphore m_semaphore; protected: diff --git a/kernel/include/kernel/LockGuard.h b/kernel/include/kernel/Lock/LockGuard.h similarity index 100% rename from kernel/include/kernel/LockGuard.h rename to kernel/include/kernel/Lock/LockGuard.h diff --git a/kernel/include/kernel/Lock/Mutex.h b/kernel/include/kernel/Lock/Mutex.h new file mode 100644 index 00000000..5ea55d1c --- /dev/null +++ b/kernel/include/kernel/Lock/Mutex.h @@ -0,0 +1,54 @@ +#pragma once + +#include +#include + +#include + +namespace Kernel +{ + + class Mutex + { + BAN_NON_COPYABLE(Mutex); + BAN_NON_MOVABLE(Mutex); + + public: + Mutex() = default; + + void lock(); + bool try_lock(); + void unlock(); + + pid_t locker() const { return m_locker; } + bool is_locked() const { return m_locker != -1; } + uint32_t lock_depth() const { return m_lock_depth; } + + private: + BAN::Atomic m_locker { -1 }; + uint32_t m_lock_depth { 0 }; + }; + + class PriorityMutex + { + BAN_NON_COPYABLE(PriorityMutex); + BAN_NON_MOVABLE(PriorityMutex); + + public: + PriorityMutex() = default; + + void lock(); + bool try_lock(); + void unlock(); + + pid_t locker() const { return m_locker; } + bool is_locked() const { return m_locker != -1; } + uint32_t lock_depth() const { return m_lock_depth; } + + private: + BAN::Atomic m_locker { -1 }; + uint32_t m_lock_depth { 0 }; + BAN::Atomic m_queue_depth { 0 }; + }; + +} diff --git a/kernel/include/kernel/Lock/SpinLock.h b/kernel/include/kernel/Lock/SpinLock.h new file mode 100644 index 00000000..17416404 --- /dev/null +++ b/kernel/include/kernel/Lock/SpinLock.h @@ -0,0 +1,54 @@ +#pragma once + +#include +#include + +#include + +namespace Kernel +{ + + class SpinLock + { + BAN_NON_COPYABLE(SpinLock); + BAN_NON_MOVABLE(SpinLock); + + public: + SpinLock() = default; + + void lock(); + bool try_lock(); + void unlock(); + + pid_t locker() const { return m_locker; } + bool is_locked() const { return m_locker != -1; } + uint32_t lock_depth() const { return is_locked(); } + + private: + BAN::Atomic m_locker { -1 }; + uintptr_t m_flags { 0 }; + }; + + class RecursiveSpinLock + { + BAN_NON_COPYABLE(RecursiveSpinLock); + BAN_NON_MOVABLE(RecursiveSpinLock); + + public: + RecursiveSpinLock() = default; + + void lock(); + bool try_lock(); + void unlock(); + + pid_t locker() const { return m_locker; } + bool is_locked() const { return m_locker != -1; } + uint32_t lock_depth() const { return m_lock_depth; } + + private: + BAN::Atomic m_locker { -1 }; + uint32_t m_lock_depth { 0 }; + uintptr_t m_flags { 0 }; + }; + +} diff --git a/kernel/include/kernel/Memory/Heap.h b/kernel/include/kernel/Memory/Heap.h index 219f7b8c..97fe3457 100644 --- a/kernel/include/kernel/Memory/Heap.h +++ b/kernel/include/kernel/Memory/Heap.h @@ -3,8 +3,8 @@ #include #include +#include #include -#include namespace Kernel { diff --git a/kernel/include/kernel/Memory/PageTable.h b/kernel/include/kernel/Memory/PageTable.h index ccd3d2ab..6f8fa24d 100644 --- a/kernel/include/kernel/Memory/PageTable.h +++ b/kernel/include/kernel/Memory/PageTable.h @@ -2,9 +2,8 @@ #include #include -#include +#include #include -#include namespace Kernel { @@ -15,6 +14,12 @@ namespace Kernel requires BAN::is_same_v; }; + template + concept with_fast_page_callback_error = requires(F func) + { + requires BAN::is_same_v>; + }; + class PageTable { public: @@ -37,19 +42,30 @@ namespace Kernel static PageTable& kernel(); static PageTable& current(); - static void map_fast_page(paddr_t); - static void unmap_fast_page(); - static constexpr vaddr_t fast_page() { return KERNEL_OFFSET; } - + public: template static void with_fast_page(paddr_t paddr, F callback) { - CriticalScope _; + s_fast_page_lock.lock(); map_fast_page(paddr); callback(); unmap_fast_page(); + s_fast_page_lock.unlock(); } + template + static BAN::ErrorOr with_fast_page(paddr_t paddr, F callback) + { + s_fast_page_lock.lock(); + map_fast_page(paddr); + auto ret = callback(); + unmap_fast_page(); + s_fast_page_lock.unlock(); + return ret; + } + + static constexpr vaddr_t fast_page() { return KERNEL_OFFSET; } + // FIXME: implement sized checks, return span, etc static void* fast_page_as_ptr(size_t offset = 0) { @@ -110,9 +126,13 @@ namespace Kernel void prepare_fast_page(); static void invalidate(vaddr_t); + static void map_fast_page(paddr_t); + static void unmap_fast_page(); + private: paddr_t m_highest_paging_struct { 0 }; mutable RecursiveSpinLock m_lock; + static SpinLock s_fast_page_lock; }; static constexpr size_t range_page_count(vaddr_t start, size_t bytes) diff --git a/kernel/include/kernel/Memory/PageTableScope.h b/kernel/include/kernel/Memory/PageTableScope.h deleted file mode 100644 index 90eb74c4..00000000 --- a/kernel/include/kernel/Memory/PageTableScope.h +++ /dev/null @@ -1,33 +0,0 @@ -#pragma once - -#include -#include -#include - -namespace Kernel -{ - - class PageTableScope - { - public: - PageTableScope(PageTable& page_table) - : m_guard(page_table) - , m_old(PageTable::current()) - , m_temp(page_table) - { - if (&m_old != &m_temp) - m_temp.load(); - } - ~PageTableScope() - { - if (&m_old != &m_temp) - m_old.load(); - } - private: - LockGuard m_guard; - CriticalScope m_scope; - PageTable& m_old; - PageTable& m_temp; - }; - -} diff --git a/kernel/include/kernel/Networking/ARPTable.h b/kernel/include/kernel/Networking/ARPTable.h index ab4e99a6..9b7a9a51 100644 --- a/kernel/include/kernel/Networking/ARPTable.h +++ b/kernel/include/kernel/Networking/ARPTable.h @@ -51,7 +51,8 @@ namespace Kernel }; private: - SpinLock m_lock; + SpinLock m_pending_lock; + SpinLock m_table_lock; BAN::HashMap m_arp_table; diff --git a/kernel/include/kernel/Networking/E1000/E1000.h b/kernel/include/kernel/Networking/E1000/E1000.h index 55a3b380..dde6edf7 100644 --- a/kernel/include/kernel/Networking/E1000/E1000.h +++ b/kernel/include/kernel/Networking/E1000/E1000.h @@ -67,6 +67,8 @@ namespace Kernel bool m_has_eerprom { false }; private: + SpinLock m_lock; + BAN::UniqPtr m_rx_buffer_region; BAN::UniqPtr m_tx_buffer_region; BAN::UniqPtr m_rx_descriptor_region; diff --git a/kernel/include/kernel/Networking/IPv4Layer.h b/kernel/include/kernel/Networking/IPv4Layer.h index 7c599132..f2bbe7d3 100644 --- a/kernel/include/kernel/Networking/IPv4Layer.h +++ b/kernel/include/kernel/Networking/IPv4Layer.h @@ -12,7 +12,6 @@ #include #include #include -#include namespace Kernel { @@ -68,7 +67,8 @@ namespace Kernel }; private: - RecursiveSpinLock m_lock; + RecursiveSpinLock m_packet_lock; + RecursiveSpinLock m_socket_lock; BAN::UniqPtr m_arp_table; Process* m_process { nullptr }; diff --git a/kernel/include/kernel/Networking/TCPSocket.h b/kernel/include/kernel/Networking/TCPSocket.h index d82b4b38..2ec8f57e 100644 --- a/kernel/include/kernel/Networking/TCPSocket.h +++ b/kernel/include/kernel/Networking/TCPSocket.h @@ -119,8 +119,7 @@ namespace Kernel uint64_t m_time_wait_start_ms { 0 }; - RecursiveSpinLock m_lock; - Semaphore m_semaphore; + Semaphore m_semaphore; BAN::Atomic m_should_ack { false }; diff --git a/kernel/include/kernel/PIC.h b/kernel/include/kernel/PIC.h index b5da20a6..723e2eb2 100644 --- a/kernel/include/kernel/PIC.h +++ b/kernel/include/kernel/PIC.h @@ -20,8 +20,11 @@ namespace Kernel private: static PIC* create(); - friend class InterruptController; + + private: + SpinLock m_lock; uint16_t m_reserved_irqs { 0 }; + friend class InterruptController; }; } diff --git a/kernel/include/kernel/Process.h b/kernel/include/kernel/Process.h index ad757159..10719c09 100644 --- a/kernel/include/kernel/Process.h +++ b/kernel/include/kernel/Process.h @@ -10,7 +10,6 @@ #include #include #include -#include #include #include @@ -222,7 +221,8 @@ namespace Kernel const pid_t m_pid; const pid_t m_parent; - mutable RecursiveSpinLock m_lock; + mutable Mutex m_big_mutex; + SpinLock m_signal_lock; BAN::String m_working_directory; BAN::Vector m_threads; diff --git a/kernel/include/kernel/Scheduler.h b/kernel/include/kernel/Scheduler.h index f4e0a100..f16d03b0 100644 --- a/kernel/include/kernel/Scheduler.h +++ b/kernel/include/kernel/Scheduler.h @@ -7,6 +7,21 @@ namespace Kernel { + class SchedulerLock + { + public: + void lock(); + void unlock(); + void unlock_all(); + pid_t locker() const; + + private: + BAN::Atomic m_locker { -1 }; + uint32_t m_lock_depth { 0 }; + + friend class Scheduler; + }; + class Scheduler { public: @@ -19,6 +34,8 @@ namespace Kernel void reschedule(); void reschedule_if_idling(); + void reschedule_current_no_save(); + void set_current_thread_sleeping(uint64_t wake_time); void block_current_thread(Semaphore*, uint64_t wake_time); @@ -29,8 +46,8 @@ namespace Kernel Thread& current_thread(); static pid_t current_tid(); - [[noreturn]] void execute_current_thread(); - [[noreturn]] void _execute_current_thread(); + BAN::ErrorOr add_thread(Thread*); + [[noreturn]] void delete_current_process_and_thread(); private: @@ -43,7 +60,8 @@ namespace Kernel void remove_and_advance_current_thread(); void advance_current_thread(); - BAN::ErrorOr add_thread(Thread*); + [[noreturn]] void execute_current_thread(); + [[noreturn]] void _execute_current_thread(); private: struct SchedulerThread @@ -57,13 +75,13 @@ namespace Kernel Semaphore* semaphore; }; + SchedulerLock m_lock; + Thread* m_idle_thread { nullptr }; BAN::LinkedList m_active_threads; BAN::LinkedList m_sleeping_threads; BAN::LinkedList::iterator m_current_thread; - - friend class Process; }; } diff --git a/kernel/include/kernel/SpinLock.h b/kernel/include/kernel/SpinLock.h deleted file mode 100644 index 4585d3f3..00000000 --- a/kernel/include/kernel/SpinLock.h +++ /dev/null @@ -1,65 +0,0 @@ -#pragma once - -#include -#include - -#include - -namespace Kernel -{ - - class SpinLock - { - BAN_NON_COPYABLE(SpinLock); - BAN_NON_MOVABLE(SpinLock); - - public: - SpinLock() = default; - void lock(); - void unlock(); - bool is_locked() const; - - uint32_t lock_depth() const { return m_locker != -1; } - - private: - BAN::Atomic m_locker = -1; - }; - - class RecursiveSpinLock - { - BAN_NON_COPYABLE(RecursiveSpinLock); - BAN_NON_MOVABLE(RecursiveSpinLock); - - public: - RecursiveSpinLock() = default; - void lock(); - void unlock(); - bool is_locked() const; - - uint32_t lock_depth() const { return m_lock_depth; } - - private: - BAN::Atomic m_locker = -1; - BAN::Atomic m_lock_depth = 0; - }; - - class RecursivePrioritySpinLock - { - BAN_NON_COPYABLE(RecursivePrioritySpinLock); - BAN_NON_MOVABLE(RecursivePrioritySpinLock); - - public: - RecursivePrioritySpinLock() = default; - void lock(); - void unlock(); - bool is_locked() const; - - uint32_t lock_depth() const { return m_lock_depth; } - - private: - BAN::Atomic m_locker = -1; - BAN::Atomic m_lock_depth = 0; - BAN::Atomic m_queue_length = 0; - }; - -} diff --git a/kernel/include/kernel/Storage/ATA/ATABus.h b/kernel/include/kernel/Storage/ATA/ATABus.h index f192613a..16991266 100644 --- a/kernel/include/kernel/Storage/ATA/ATABus.h +++ b/kernel/include/kernel/Storage/ATA/ATABus.h @@ -4,7 +4,7 @@ #include #include #include -#include +#include namespace Kernel { @@ -51,7 +51,7 @@ namespace Kernel private: const uint16_t m_base; const uint16_t m_ctrl; - SpinLock m_lock; + Mutex m_mutex; volatile bool m_has_got_irq { false }; diff --git a/kernel/include/kernel/Storage/DiskCache.h b/kernel/include/kernel/Storage/DiskCache.h index 006a113f..a5a79313 100644 --- a/kernel/include/kernel/Storage/DiskCache.h +++ b/kernel/include/kernel/Storage/DiskCache.h @@ -3,7 +3,6 @@ #include #include #include -#include namespace Kernel { diff --git a/kernel/include/kernel/Storage/NVMe/Queue.h b/kernel/include/kernel/Storage/NVMe/Queue.h index 357a3a0a..964cf616 100644 --- a/kernel/include/kernel/Storage/NVMe/Queue.h +++ b/kernel/include/kernel/Storage/NVMe/Queue.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -20,7 +21,7 @@ namespace Kernel virtual void handle_irq() final override; private: - SpinLock m_lock; + Mutex m_mutex; BAN::UniqPtr m_completion_queue; BAN::UniqPtr m_submission_queue; volatile NVMe::DoorbellRegisters& m_doorbell; diff --git a/kernel/include/kernel/Storage/StorageDevice.h b/kernel/include/kernel/Storage/StorageDevice.h index aa8dc7f4..69dfcba8 100644 --- a/kernel/include/kernel/Storage/StorageDevice.h +++ b/kernel/include/kernel/Storage/StorageDevice.h @@ -44,7 +44,7 @@ namespace Kernel virtual bool has_error_impl() const override { return false; } private: - SpinLock m_lock; + Mutex m_mutex; BAN::Optional m_disk_cache; BAN::Vector> m_partitions; diff --git a/kernel/include/kernel/Terminal/Serial.h b/kernel/include/kernel/Terminal/Serial.h index d666bbf4..5e13af5c 100644 --- a/kernel/include/kernel/Terminal/Serial.h +++ b/kernel/include/kernel/Terminal/Serial.h @@ -59,6 +59,7 @@ namespace Kernel bool initialize(); private: + SpinLock m_lock; BAN::String m_name; Serial m_serial; BAN::CircularQueue m_input; diff --git a/kernel/include/kernel/Terminal/TTY.h b/kernel/include/kernel/Terminal/TTY.h index e88b2915..25e261c1 100644 --- a/kernel/include/kernel/Terminal/TTY.h +++ b/kernel/include/kernel/Terminal/TTY.h @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include diff --git a/kernel/include/kernel/Terminal/VirtualTTY.h b/kernel/include/kernel/Terminal/VirtualTTY.h index e8b7a117..a4ccb4d0 100644 --- a/kernel/include/kernel/Terminal/VirtualTTY.h +++ b/kernel/include/kernel/Terminal/VirtualTTY.h @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -70,6 +69,8 @@ namespace Kernel private: BAN::String m_name; + RecursiveSpinLock m_write_lock; + State m_state { State::Normal }; AnsiState m_ansi_state { }; UTF8State m_utf8_state { }; diff --git a/kernel/include/kernel/Thread.h b/kernel/include/kernel/Thread.h index dd5e4648..8f4a29fb 100644 --- a/kernel/include/kernel/Thread.h +++ b/kernel/include/kernel/Thread.h @@ -113,6 +113,8 @@ namespace Kernel Process* m_process { nullptr }; bool m_is_userspace { false }; + mutable RecursiveSpinLock m_lock; + uintptr_t* m_return_rsp { nullptr }; uintptr_t* m_return_rip { nullptr }; diff --git a/kernel/include/kernel/Timer/HPET.h b/kernel/include/kernel/Timer/HPET.h index f462b9dc..9bfec0d1 100644 --- a/kernel/include/kernel/Timer/HPET.h +++ b/kernel/include/kernel/Timer/HPET.h @@ -30,6 +30,8 @@ namespace Kernel uint64_t read_main_counter() const; private: + mutable SpinLock m_lock; + bool m_is_64bit { false }; uint64_t m_last_ticks { 0 }; diff --git a/kernel/kernel/ACPI.cpp b/kernel/kernel/ACPI.cpp index ff397d97..fabd8e38 100644 --- a/kernel/kernel/ACPI.cpp +++ b/kernel/kernel/ACPI.cpp @@ -99,33 +99,39 @@ namespace Kernel if (rsdp->revision >= 2) { - PageTable::map_fast_page(rsdp->xsdt_address & PAGE_ADDR_MASK); - auto& xsdt = PageTable::fast_page_as(rsdp->xsdt_address % PAGE_SIZE); - BAN::ScopeGuard _([] { PageTable::unmap_fast_page(); }); + TRY(PageTable::with_fast_page(rsdp->xsdt_address & PAGE_ADDR_MASK, + [&]() -> BAN::ErrorOr + { + auto& xsdt = PageTable::fast_page_as(rsdp->xsdt_address % PAGE_SIZE); + if (memcmp(xsdt.signature, "XSDT", 4) != 0) + return BAN::Error::from_error_code(ErrorCode::ACPI_RootInvalid); + if (!is_valid_std_header(&xsdt)) + return BAN::Error::from_error_code(ErrorCode::ACPI_RootInvalid); - if (memcmp(xsdt.signature, "XSDT", 4) != 0) - return BAN::Error::from_error_code(ErrorCode::ACPI_RootInvalid); - if (!is_valid_std_header(&xsdt)) - return BAN::Error::from_error_code(ErrorCode::ACPI_RootInvalid); - - m_header_table_paddr = rsdp->xsdt_address + offsetof(XSDT, entries); - m_entry_size = 8; - root_entry_count = (xsdt.length - sizeof(SDTHeader)) / 8; + m_header_table_paddr = rsdp->xsdt_address + offsetof(XSDT, entries); + m_entry_size = 8; + root_entry_count = (xsdt.length - sizeof(SDTHeader)) / 8; + return {}; + } + )); } else { - PageTable::map_fast_page(rsdp->rsdt_address & PAGE_ADDR_MASK); - auto& rsdt = PageTable::fast_page_as(rsdp->rsdt_address % PAGE_SIZE); - BAN::ScopeGuard _([] { PageTable::unmap_fast_page(); }); + TRY(PageTable::with_fast_page(rsdp->rsdt_address & PAGE_ADDR_MASK, + [&]() -> BAN::ErrorOr + { + auto& rsdt = PageTable::fast_page_as(rsdp->rsdt_address % PAGE_SIZE); + if (memcmp(rsdt.signature, "RSDT", 4) != 0) + return BAN::Error::from_error_code(ErrorCode::ACPI_RootInvalid); + if (!is_valid_std_header(&rsdt)) + return BAN::Error::from_error_code(ErrorCode::ACPI_RootInvalid); - if (memcmp(rsdt.signature, "RSDT", 4) != 0) - return BAN::Error::from_error_code(ErrorCode::ACPI_RootInvalid); - if (!is_valid_std_header(&rsdt)) - return BAN::Error::from_error_code(ErrorCode::ACPI_RootInvalid); - - m_header_table_paddr = rsdp->rsdt_address + offsetof(RSDT, entries); - m_entry_size = 4; - root_entry_count = (rsdt.length - sizeof(SDTHeader)) / 4; + m_header_table_paddr = rsdp->rsdt_address + offsetof(RSDT, entries); + m_entry_size = 4; + root_entry_count = (rsdt.length - sizeof(SDTHeader)) / 4; + return {}; + } + )); } size_t needed_pages = range_page_count(m_header_table_paddr, root_entry_count * m_entry_size); @@ -144,9 +150,10 @@ namespace Kernel auto map_header = [](paddr_t header_paddr) -> vaddr_t { - PageTable::map_fast_page(header_paddr & PAGE_ADDR_MASK); - size_t header_length = PageTable::fast_page_as(header_paddr % PAGE_SIZE).length; - PageTable::unmap_fast_page(); + size_t header_length; + PageTable::with_fast_page(header_paddr & PAGE_ADDR_MASK, [&] { + header_length = PageTable::fast_page_as(header_paddr % PAGE_SIZE).length; + }); size_t needed_pages = range_page_count(header_paddr, header_length); vaddr_t page_vaddr = PageTable::kernel().reserve_free_contiguous_pages(needed_pages, KERNEL_OFFSET); diff --git a/kernel/kernel/APIC.cpp b/kernel/kernel/APIC.cpp index 6098f1e1..6fcbc2b0 100644 --- a/kernel/kernel/APIC.cpp +++ b/kernel/kernel/APIC.cpp @@ -1,9 +1,10 @@ #include -#include #include #include #include +#include #include +#include #include #include @@ -223,7 +224,7 @@ namespace Kernel void APIC::enable_irq(uint8_t irq) { - CriticalScope _; + LockGuard _(m_lock); uint32_t gsi = m_irq_overrides[irq]; @@ -268,7 +269,7 @@ namespace Kernel BAN::ErrorOr APIC::reserve_irq(uint8_t irq) { - CriticalScope _; + LockGuard _(m_lock); uint32_t gsi = m_irq_overrides[irq]; @@ -301,7 +302,7 @@ namespace Kernel BAN::Optional APIC::get_free_irq() { - CriticalScope _; + LockGuard _(m_lock); for (int irq = 0; irq <= 0xFF; irq++) { uint32_t gsi = m_irq_overrides[irq]; diff --git a/kernel/kernel/Debug.cpp b/kernel/kernel/Debug.cpp index fd446579..53f9b572 100644 --- a/kernel/kernel/Debug.cpp +++ b/kernel/kernel/Debug.cpp @@ -1,7 +1,6 @@ #include #include #include -#include #include #include #include @@ -13,6 +12,8 @@ extern TerminalDriver* g_terminal_driver; namespace Debug { + Kernel::RecursiveSpinLock s_debug_lock; + void dump_stack_trace() { using namespace Kernel; @@ -120,18 +121,4 @@ namespace Debug BAN::Formatter::print(Debug::putchar, "[{5}.{3}] {}:{}: ", ms_since_boot / 1000, ms_since_boot % 1000, file, line); } - static Kernel::RecursiveSpinLock s_debug_lock; - - void DebugLock::lock() - { - if (Kernel::interrupts_enabled()) - s_debug_lock.lock(); - } - - void DebugLock::unlock() - { - if (Kernel::interrupts_enabled()) - s_debug_lock.unlock(); - } - } diff --git a/kernel/kernel/Device/DebugDevice.cpp b/kernel/kernel/Device/DebugDevice.cpp index 35615204..a9567636 100644 --- a/kernel/kernel/Device/DebugDevice.cpp +++ b/kernel/kernel/Device/DebugDevice.cpp @@ -20,7 +20,7 @@ namespace Kernel BAN::ErrorOr DebugDevice::write_impl(off_t, BAN::ConstByteSpan buffer) { auto ms_since_boot = SystemTimer::get().ms_since_boot(); - Debug::DebugLock::lock(); + Debug::s_debug_lock.lock(); BAN::Formatter::print(Debug::putchar, "[{5}.{3}] {}: ", ms_since_boot / 1000, ms_since_boot % 1000, @@ -28,7 +28,7 @@ namespace Kernel ); for (size_t i = 0; i < buffer.size(); i++) Debug::putchar(buffer[i]); - Debug::DebugLock::unlock(); + Debug::s_debug_lock.unlock(); return buffer.size(); } diff --git a/kernel/kernel/FS/DevFS/FileSystem.cpp b/kernel/kernel/FS/DevFS/FileSystem.cpp index 2e615d20..a774710d 100644 --- a/kernel/kernel/FS/DevFS/FileSystem.cpp +++ b/kernel/kernel/FS/DevFS/FileSystem.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/kernel/kernel/FS/Ext2/FileSystem.cpp b/kernel/kernel/FS/Ext2/FileSystem.cpp index 88599076..0b3c91a0 100644 --- a/kernel/kernel/FS/Ext2/FileSystem.cpp +++ b/kernel/kernel/FS/Ext2/FileSystem.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #define EXT2_DEBUG_PRINT 0 #define EXT2_VERIFY_INODE 0 @@ -139,7 +139,7 @@ namespace Kernel BAN::ErrorOr Ext2FS::create_inode(const Ext2::Inode& ext2_inode) { - LockGuard _(m_lock); + LockGuard _(m_mutex); ASSERT(ext2_inode.size == 0); @@ -218,7 +218,7 @@ namespace Kernel void Ext2FS::delete_inode(uint32_t ino) { - LockGuard _(m_lock); + LockGuard _(m_mutex); ASSERT(ino >= superblock().first_ino); ASSERT(ino <= superblock().inodes_count); @@ -271,7 +271,7 @@ namespace Kernel void Ext2FS::read_block(uint32_t block, BlockBufferWrapper& buffer) { - LockGuard _(m_lock); + LockGuard _(m_mutex); const uint32_t sector_size = m_block_device->blksize(); const uint32_t block_size = this->block_size(); @@ -284,7 +284,7 @@ namespace Kernel void Ext2FS::write_block(uint32_t block, const BlockBufferWrapper& buffer) { - LockGuard _(m_lock); + LockGuard _(m_mutex); const uint32_t sector_size = m_block_device->blksize(); const uint32_t block_size = this->block_size(); @@ -297,7 +297,7 @@ namespace Kernel void Ext2FS::sync_superblock() { - LockGuard _(m_lock); + LockGuard _(m_mutex); const uint32_t sector_size = m_block_device->blksize(); ASSERT(1024 % sector_size == 0); @@ -322,13 +322,13 @@ namespace Kernel Ext2FS::BlockBufferWrapper Ext2FS::get_block_buffer() { - LockGuard _(m_lock); + LockGuard _(m_mutex); return m_buffer_manager.get_buffer(); } BAN::ErrorOr Ext2FS::reserve_free_block(uint32_t primary_bgd) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (m_superblock.r_blocks_count >= m_superblock.free_blocks_count) return BAN::Error::from_errno(ENOSPC); @@ -389,7 +389,7 @@ namespace Kernel void Ext2FS::release_block(uint32_t block) { - LockGuard _(m_lock); + LockGuard _(m_mutex); ASSERT(block >= m_superblock.first_data_block); ASSERT(block < m_superblock.blocks_count); @@ -422,7 +422,7 @@ namespace Kernel Ext2FS::BlockLocation Ext2FS::locate_inode(uint32_t ino) { - LockGuard _(m_lock); + LockGuard _(m_mutex); ASSERT(ino <= superblock().inodes_count); @@ -464,7 +464,7 @@ namespace Kernel Ext2FS::BlockLocation Ext2FS::locate_block_group_descriptior(uint32_t group_index) { - LockGuard _(m_lock); + LockGuard _(m_mutex); const uint32_t block_size = this->block_size(); diff --git a/kernel/kernel/FS/Inode.cpp b/kernel/kernel/FS/Inode.cpp index 74702e49..b7341f0f 100644 --- a/kernel/kernel/FS/Inode.cpp +++ b/kernel/kernel/FS/Inode.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include @@ -58,13 +58,13 @@ namespace Kernel void Inode::on_close() { - LockGuard _(m_lock); + LockGuard _(m_mutex); on_close_impl(); } BAN::ErrorOr> Inode::find_inode(BAN::StringView name) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (!mode().ifdir()) return BAN::Error::from_errno(ENOTDIR); return find_inode_impl(name); @@ -72,7 +72,7 @@ namespace Kernel BAN::ErrorOr Inode::list_next_inodes(off_t offset, DirectoryEntryList* list, size_t list_len) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (!mode().ifdir()) return BAN::Error::from_errno(ENOTDIR); return list_next_inodes_impl(offset, list, list_len); @@ -80,7 +80,7 @@ namespace Kernel BAN::ErrorOr Inode::create_file(BAN::StringView name, mode_t mode, uid_t uid, gid_t gid) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (!this->mode().ifdir()) return BAN::Error::from_errno(ENOTDIR); if (Mode(mode).ifdir()) @@ -90,7 +90,7 @@ namespace Kernel BAN::ErrorOr Inode::create_directory(BAN::StringView name, mode_t mode, uid_t uid, gid_t gid) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (!this->mode().ifdir()) return BAN::Error::from_errno(ENOTDIR); if (!Mode(mode).ifdir()) @@ -100,7 +100,7 @@ namespace Kernel BAN::ErrorOr Inode::unlink(BAN::StringView name) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (!mode().ifdir()) return BAN::Error::from_errno(ENOTDIR); if (name == "."sv || name == ".."sv) @@ -110,7 +110,7 @@ namespace Kernel BAN::ErrorOr Inode::link_target() { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (!mode().iflnk()) return BAN::Error::from_errno(EINVAL); return link_target_impl(); @@ -118,7 +118,7 @@ namespace Kernel BAN::ErrorOr Inode::accept(sockaddr* address, socklen_t* address_len) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (!mode().ifsock()) return BAN::Error::from_errno(ENOTSOCK); return accept_impl(address, address_len); @@ -126,7 +126,7 @@ namespace Kernel BAN::ErrorOr Inode::bind(const sockaddr* address, socklen_t address_len) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (!mode().ifsock()) return BAN::Error::from_errno(ENOTSOCK); return bind_impl(address, address_len); @@ -134,7 +134,7 @@ namespace Kernel BAN::ErrorOr Inode::connect(const sockaddr* address, socklen_t address_len) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (!mode().ifsock()) return BAN::Error::from_errno(ENOTSOCK); return connect_impl(address, address_len); @@ -142,7 +142,7 @@ namespace Kernel BAN::ErrorOr Inode::listen(int backlog) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (!mode().ifsock()) return BAN::Error::from_errno(ENOTSOCK); return listen_impl(backlog); @@ -150,7 +150,7 @@ namespace Kernel BAN::ErrorOr Inode::sendto(BAN::ConstByteSpan message, const sockaddr* address, socklen_t address_len) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (!mode().ifsock()) return BAN::Error::from_errno(ENOTSOCK); return sendto_impl(message, address, address_len); @@ -158,7 +158,7 @@ namespace Kernel BAN::ErrorOr Inode::recvfrom(BAN::ByteSpan buffer, sockaddr* address, socklen_t* address_len) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (!mode().ifsock()) return BAN::Error::from_errno(ENOTSOCK); return recvfrom_impl(buffer, address, address_len); @@ -166,7 +166,7 @@ namespace Kernel BAN::ErrorOr Inode::read(off_t offset, BAN::ByteSpan buffer) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (mode().ifdir()) return BAN::Error::from_errno(EISDIR); return read_impl(offset, buffer); @@ -174,7 +174,7 @@ namespace Kernel BAN::ErrorOr Inode::write(off_t offset, BAN::ConstByteSpan buffer) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (mode().ifdir()) return BAN::Error::from_errno(EISDIR); return write_impl(offset, buffer); @@ -182,7 +182,7 @@ namespace Kernel BAN::ErrorOr Inode::truncate(size_t size) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (mode().ifdir()) return BAN::Error::from_errno(EISDIR); return truncate_impl(size); @@ -191,37 +191,37 @@ namespace Kernel BAN::ErrorOr Inode::chmod(mode_t mode) { ASSERT((mode & Inode::Mode::TYPE_MASK) == 0); - LockGuard _(m_lock); + LockGuard _(m_mutex); return chmod_impl(mode); } BAN::ErrorOr Inode::chown(uid_t uid, gid_t gid) { - LockGuard _(m_lock); + LockGuard _(m_mutex); return chown_impl(uid, gid); } bool Inode::can_read() const { - LockGuard _(m_lock); + LockGuard _(m_mutex); return can_read_impl(); } bool Inode::can_write() const { - LockGuard _(m_lock); + LockGuard _(m_mutex); return can_write_impl(); } bool Inode::has_error() const { - LockGuard _(m_lock); + LockGuard _(m_mutex); return has_error_impl(); } BAN::ErrorOr Inode::ioctl(int request, void* arg) { - LockGuard _(m_lock); + LockGuard _(m_mutex); return ioctl_impl(request, arg); } diff --git a/kernel/kernel/FS/Pipe.cpp b/kernel/kernel/FS/Pipe.cpp index af70a75f..8b10f5a6 100644 --- a/kernel/kernel/FS/Pipe.cpp +++ b/kernel/kernel/FS/Pipe.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include @@ -26,14 +26,14 @@ namespace Kernel void Pipe::clone_writing() { - LockGuard _(m_lock); + LockGuard _(m_mutex); ASSERT(m_writing_count > 0); m_writing_count++; } void Pipe::close_writing() { - LockGuard _(m_lock); + LockGuard _(m_mutex); ASSERT(m_writing_count > 0); m_writing_count--; if (m_writing_count == 0) @@ -42,12 +42,11 @@ namespace Kernel BAN::ErrorOr Pipe::read_impl(off_t, BAN::ByteSpan buffer) { - LockGuard _(m_lock); while (m_buffer.empty()) { if (m_writing_count == 0) return 0; - LockFreeGuard lock_free(m_lock); + LockFreeGuard lock_free(m_mutex); TRY(Thread::current().block_or_eintr_indefinite(m_semaphore)); } @@ -66,8 +65,6 @@ namespace Kernel BAN::ErrorOr Pipe::write_impl(off_t, BAN::ConstByteSpan buffer) { - LockGuard _(m_lock); - size_t old_size = m_buffer.size(); TRY(m_buffer.resize(old_size + buffer.size())); diff --git a/kernel/kernel/FS/ProcFS/FileSystem.cpp b/kernel/kernel/FS/ProcFS/FileSystem.cpp index d5400582..059742d8 100644 --- a/kernel/kernel/FS/ProcFS/FileSystem.cpp +++ b/kernel/kernel/FS/ProcFS/FileSystem.cpp @@ -1,6 +1,5 @@ #include #include -#include namespace Kernel { diff --git a/kernel/kernel/FS/TmpFS/FileSystem.cpp b/kernel/kernel/FS/TmpFS/FileSystem.cpp index 582afb35..bc94ac50 100644 --- a/kernel/kernel/FS/TmpFS/FileSystem.cpp +++ b/kernel/kernel/FS/TmpFS/FileSystem.cpp @@ -66,7 +66,7 @@ namespace Kernel BAN::ErrorOr> TmpFileSystem::open_inode(ino_t ino) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (m_inode_cache.contains(ino)) return m_inode_cache[ino]; @@ -85,7 +85,7 @@ namespace Kernel BAN::ErrorOr TmpFileSystem::add_to_cache(BAN::RefPtr inode) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (!m_inode_cache.contains(inode->ino())) TRY(m_inode_cache.insert(inode->ino(), inode)); @@ -94,7 +94,7 @@ namespace Kernel void TmpFileSystem::remove_from_cache(BAN::RefPtr inode) { - LockGuard _(m_lock); + LockGuard _(m_mutex); ASSERT(m_inode_cache.contains(inode->ino())); m_inode_cache.remove(inode->ino()); @@ -102,7 +102,7 @@ namespace Kernel void TmpFileSystem::read_inode(ino_t ino, TmpInodeInfo& out) { - LockGuard _(m_lock); + LockGuard _(m_mutex); auto inode_location = find_inode(ino); PageTable::with_fast_page(inode_location.paddr, [&] { @@ -112,7 +112,7 @@ namespace Kernel void TmpFileSystem::write_inode(ino_t ino, const TmpInodeInfo& info) { - LockGuard _(m_lock); + LockGuard _(m_mutex); auto inode_location = find_inode(ino); PageTable::with_fast_page(inode_location.paddr, [&] { @@ -123,7 +123,7 @@ namespace Kernel void TmpFileSystem::delete_inode(ino_t ino) { - LockGuard _(m_lock); + LockGuard _(m_mutex); auto inode_location = find_inode(ino); PageTable::with_fast_page(inode_location.paddr, [&] { @@ -138,7 +138,7 @@ namespace Kernel BAN::ErrorOr TmpFileSystem::allocate_inode(const TmpInodeInfo& info) { - LockGuard _(m_lock); + LockGuard _(m_mutex); constexpr size_t inodes_per_page = PAGE_SIZE / sizeof(TmpInodeInfo); @@ -164,7 +164,7 @@ namespace Kernel TmpFileSystem::InodeLocation TmpFileSystem::find_inode(ino_t ino) { - LockGuard _(m_lock); + LockGuard _(m_mutex); ASSERT_GTE(ino, first_inode); ASSERT_LT(ino, max_inodes); @@ -182,7 +182,7 @@ namespace Kernel void TmpFileSystem::free_block(size_t index) { - LockGuard _(m_lock); + LockGuard _(m_mutex); constexpr size_t addresses_per_page = PAGE_SIZE / sizeof(PageInfo); @@ -204,7 +204,7 @@ namespace Kernel BAN::ErrorOr TmpFileSystem::allocate_block() { - LockGuard _(m_lock); + LockGuard _(m_mutex); size_t result = first_data_page; TRY(for_each_indirect_paddr_allocating(m_data_pages, [&] (paddr_t, bool allocated) { @@ -218,7 +218,7 @@ namespace Kernel paddr_t TmpFileSystem::find_block(size_t index) { - LockGuard _(m_lock); + LockGuard _(m_mutex); ASSERT_GT(index, 0); return find_indirect(m_data_pages, index - first_data_page, 3); @@ -226,7 +226,7 @@ namespace Kernel paddr_t TmpFileSystem::find_indirect(PageInfo root, size_t index, size_t depth) { - LockGuard _(m_lock); + LockGuard _(m_mutex); ASSERT(root.flags() & PageInfo::Flags::Present); if (depth == 0) @@ -257,7 +257,7 @@ namespace Kernel template BAN::ErrorOr TmpFileSystem::for_each_indirect_paddr_allocating_internal(PageInfo page_info, F callback, size_t depth) { - LockGuard _(m_lock); + LockGuard _(m_mutex); ASSERT(page_info.flags() & PageInfo::Flags::Present); if (depth == 0) @@ -316,7 +316,7 @@ namespace Kernel template BAN::ErrorOr TmpFileSystem::for_each_indirect_paddr_allocating(PageInfo page_info, F callback, size_t depth) { - LockGuard _(m_lock); + LockGuard _(m_mutex); BAN::Iteration result = TRY(for_each_indirect_paddr_allocating_internal(page_info, callback, depth)); ASSERT(result == BAN::Iteration::Break); diff --git a/kernel/kernel/FS/VirtualFileSystem.cpp b/kernel/kernel/FS/VirtualFileSystem.cpp index 36ba3426..f41661b4 100644 --- a/kernel/kernel/FS/VirtualFileSystem.cpp +++ b/kernel/kernel/FS/VirtualFileSystem.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include namespace Kernel @@ -44,6 +44,8 @@ namespace Kernel BAN::ErrorOr VirtualFileSystem::mount(const Credentials& credentials, BAN::StringView block_device_path, BAN::StringView target) { + LockGuard _(m_mutex); + auto block_device_file = TRY(file_from_absolute_path(credentials, block_device_path, true)); if (!block_device_file.inode->is_device()) return BAN::Error::from_errno(ENOTBLK); @@ -63,15 +65,14 @@ namespace Kernel if (!file.inode->mode().ifdir()) return BAN::Error::from_errno(ENOTDIR); - LockGuard _(m_lock); + LockGuard _(m_mutex); TRY(m_mount_points.push_back({ file, file_system })); - return {}; } VirtualFileSystem::MountPoint* VirtualFileSystem::mount_from_host_inode(BAN::RefPtr inode) { - ASSERT(m_lock.is_locked()); + LockGuard _(m_mutex); for (MountPoint& mount : m_mount_points) if (*mount.host.inode == *inode) return &mount; @@ -80,7 +81,7 @@ namespace Kernel VirtualFileSystem::MountPoint* VirtualFileSystem::mount_from_root_inode(BAN::RefPtr inode) { - ASSERT(m_lock.is_locked()); + LockGuard _(m_mutex); for (MountPoint& mount : m_mount_points) if (*mount.target->root_inode() == *inode) return &mount; @@ -89,7 +90,7 @@ namespace Kernel BAN::ErrorOr VirtualFileSystem::file_from_absolute_path(const Credentials& credentials, BAN::StringView path, int flags) { - LockGuard _(m_lock); + LockGuard _(m_mutex); ASSERT(path.front() == '/'); diff --git a/kernel/kernel/Input/KeyboardLayout.cpp b/kernel/kernel/Input/KeyboardLayout.cpp index c8b6901d..2a0f76e3 100644 --- a/kernel/kernel/Input/KeyboardLayout.cpp +++ b/kernel/kernel/Input/KeyboardLayout.cpp @@ -1,7 +1,7 @@ #include -#include #include #include +#include #include @@ -74,6 +74,7 @@ namespace Kernel::Input Key KeyboardLayout::get_key_from_event(KeyEvent event) { + LockGuard _(m_lock); if (event.shift()) return m_keycode_to_key_shift[event.keycode]; if (event.ralt()) @@ -256,7 +257,7 @@ namespace Kernel::Input } } - CriticalScope _; + LockGuard _(m_lock); for (size_t i = 0; i < new_layout->m_keycode_to_key_normal.size(); i++) if (new_layout->m_keycode_to_key_normal[i] != Key::None) diff --git a/kernel/kernel/Input/PS2/Controller.cpp b/kernel/kernel/Input/PS2/Controller.cpp index 9f6fc1d5..90757c06 100644 --- a/kernel/kernel/Input/PS2/Controller.cpp +++ b/kernel/kernel/Input/PS2/Controller.cpp @@ -22,7 +22,7 @@ namespace Kernel::Input BAN::ErrorOr PS2Controller::send_byte(uint16_t port, uint8_t byte) { ASSERT(interrupts_enabled()); - LockGuard _(m_lock); + LockGuard _(m_mutex); uint64_t timeout = SystemTimer::get().ms_since_boot() + s_ps2_timeout_ms; while (SystemTimer::get().ms_since_boot() < timeout) { @@ -37,7 +37,7 @@ namespace Kernel::Input BAN::ErrorOr PS2Controller::read_byte() { ASSERT(interrupts_enabled()); - LockGuard _(m_lock); + LockGuard _(m_mutex); uint64_t timeout = SystemTimer::get().ms_since_boot() + s_ps2_timeout_ms; while (SystemTimer::get().ms_since_boot() < timeout) { @@ -50,14 +50,14 @@ namespace Kernel::Input BAN::ErrorOr PS2Controller::send_command(PS2::Command command) { - LockGuard _(m_lock); + LockGuard _(m_mutex); TRY(send_byte(PS2::IOPort::COMMAND, command)); return {}; } BAN::ErrorOr PS2Controller::send_command(PS2::Command command, uint8_t data) { - LockGuard _(m_lock); + LockGuard _(m_mutex); TRY(send_byte(PS2::IOPort::COMMAND, command)); TRY(send_byte(PS2::IOPort::DATA, data)); return {}; @@ -65,7 +65,7 @@ namespace Kernel::Input BAN::ErrorOr PS2Controller::device_send_byte(uint8_t device_index, uint8_t byte) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (device_index == 1) TRY(send_byte(PS2::IOPort::COMMAND, PS2::Command::WRITE_TO_SECOND_PORT)); TRY(send_byte(PS2::IOPort::DATA, byte)); @@ -74,7 +74,7 @@ namespace Kernel::Input BAN::ErrorOr PS2Controller::device_send_byte_and_wait_ack(uint8_t device_index, uint8_t byte) { - LockGuard _(m_lock); + LockGuard _(m_mutex); for (;;) { TRY(device_send_byte(device_index, byte)); @@ -101,8 +101,7 @@ namespace Kernel::Input bool PS2Controller::append_command_queue(PS2Device* device, uint8_t command, uint8_t response_size) { - // NOTE: command queue push/pop must be done without interrupts - CriticalScope _; + LockGuard _(m_cmd_lock); if (m_command_queue.size() + 1 >= m_command_queue.capacity()) { dprintln("PS/2 command queue full"); @@ -121,8 +120,7 @@ namespace Kernel::Input bool PS2Controller::append_command_queue(PS2Device* device, uint8_t command, uint8_t data, uint8_t response_size) { - // NOTE: command queue push/pop must be done without interrupts - CriticalScope _; + LockGuard _(m_cmd_lock); if (m_command_queue.size() + 1 >= m_command_queue.capacity()) { dprintln("PS/2 command queue full"); @@ -143,6 +141,9 @@ namespace Kernel::Input { ASSERT(interrupts_enabled()); + // NOTE: CircularQueue reads don't need locking, as long as + // we can guarantee that read element is not popped + if (m_command_queue.empty()) return; auto& command = m_command_queue.front(); @@ -152,6 +153,8 @@ namespace Kernel::Input { dwarnln_if(DEBUG_PS2, "Command timedout"); m_devices[command.device_index]->command_timedout(command.out_data, command.out_count); + + LockGuard _(m_cmd_lock); m_command_queue.pop(); } return; diff --git a/kernel/kernel/Input/PS2/Keyboard.cpp b/kernel/kernel/Input/PS2/Keyboard.cpp index ae028f35..65accbb7 100644 --- a/kernel/kernel/Input/PS2/Keyboard.cpp +++ b/kernel/kernel/Input/PS2/Keyboard.cpp @@ -1,5 +1,4 @@ #include -#include #include #include #include @@ -165,6 +164,7 @@ namespace Kernel::Input event.modifier = m_modifiers | (released ? 0 : KeyEvent::Modifier::Pressed); event.keycode = keycode.value(); + LockGuard _(m_event_lock); if (m_event_queue.full()) { dwarnln("PS/2 event queue full"); @@ -197,7 +197,7 @@ namespace Kernel::Input if (m_event_queue.empty()) TRY(Thread::current().block_or_eintr_indefinite(m_semaphore)); - CriticalScope _; + LockGuard _(m_event_lock); if (m_event_queue.empty()) continue; diff --git a/kernel/kernel/Input/PS2/Mouse.cpp b/kernel/kernel/Input/PS2/Mouse.cpp index dfcc3ac0..de603444 100644 --- a/kernel/kernel/Input/PS2/Mouse.cpp +++ b/kernel/kernel/Input/PS2/Mouse.cpp @@ -1,5 +1,4 @@ #include -#include #include #include #include @@ -158,6 +157,7 @@ namespace Kernel::Input event.scroll_event.scroll = rel_z; } + LockGuard _(m_event_lock); for (int i = 0; i < event_count; i++) { if (m_event_queue.full()) @@ -181,7 +181,7 @@ namespace Kernel::Input if (m_event_queue.empty()) TRY(Thread::current().block_or_eintr_indefinite(m_semaphore)); - CriticalScope _; + LockGuard _(m_event_lock); if (m_event_queue.empty()) continue; diff --git a/kernel/kernel/Lock/Mutex.cpp b/kernel/kernel/Lock/Mutex.cpp new file mode 100644 index 00000000..4631449d --- /dev/null +++ b/kernel/kernel/Lock/Mutex.cpp @@ -0,0 +1,68 @@ +#include +#include + +namespace Kernel +{ + + void Mutex::lock() + { + auto tid = Scheduler::current_tid(); + if (tid != m_locker) + while (!m_locker.compare_exchange(-1, tid)) + Scheduler::get().reschedule(); + m_lock_depth++; + } + + bool Mutex::try_lock() + { + auto tid = Scheduler::current_tid(); + if (tid != m_locker) + if (!m_locker.compare_exchange(-1, tid)) + return false; + m_lock_depth++; + return true; + } + + void Mutex::unlock() + { + ASSERT_EQ(m_locker.load(), Scheduler::current_tid()); + if (--m_lock_depth == 0) + m_locker = -1; + } + + void PriorityMutex::lock() + { + const auto tid = Scheduler::current_tid(); + const bool has_priority = tid ? !Thread::current().is_userspace() : true; + if (has_priority) + m_queue_depth++; + if (tid != m_locker) + while ((!has_priority && m_queue_depth > 0) || !m_locker.compare_exchange(-1, tid)) + asm volatile("pause"); + m_lock_depth++; + } + + bool PriorityMutex::try_lock() + { + const auto tid = Scheduler::current_tid(); + const bool has_priority = tid ? !Thread::current().is_userspace() : true; + if (tid != m_locker) + while ((!has_priority && m_queue_depth > 0) || !m_locker.compare_exchange(-1, tid)) + return false; + if (has_priority) + m_queue_depth++; + m_lock_depth++; + return true; + } + + void PriorityMutex::unlock() + { + const auto tid = Scheduler::current_tid(); + const bool has_priority = tid ? !Thread::current().is_userspace() : true; + if (has_priority) + m_queue_depth--; + if (--m_lock_depth) + m_locker = -1; + } + +} diff --git a/kernel/kernel/Lock/SpinLock.cpp b/kernel/kernel/Lock/SpinLock.cpp new file mode 100644 index 00000000..e9b61479 --- /dev/null +++ b/kernel/kernel/Lock/SpinLock.cpp @@ -0,0 +1,80 @@ +#include +#include + +namespace Kernel +{ + + static inline uintptr_t get_flags_and_disable_interrupts() + { + uintptr_t flags; + asm volatile("pushf; cli; pop %0" : "=r"(flags) :: "memory"); + return flags; + } + + static inline void restore_flags(uintptr_t flags) + { + asm volatile("push %0; popf" :: "rm"(flags) : "memory", "cc"); + } + + void SpinLock::lock() + { + const auto tid = Scheduler::current_tid(); + ASSERT_NEQ(m_locker.load(), tid); + while (!m_locker.compare_exchange(-1, tid)) + __builtin_ia32_pause(); + m_flags = get_flags_and_disable_interrupts(); + } + + bool SpinLock::try_lock() + { + const auto tid = Scheduler::current_tid(); + ASSERT_NEQ(m_locker.load(), tid); + if (!m_locker.compare_exchange(-1, tid)) + return false; + m_flags = get_flags_and_disable_interrupts(); + return true; + } + + void SpinLock::unlock() + { + ASSERT_EQ(m_locker.load(), Scheduler::current_tid()); + restore_flags(m_flags); + m_locker = -1; + } + + void RecursiveSpinLock::lock() + { + auto tid = Scheduler::current_tid(); + if (m_locker != tid) + { + while (!m_locker.compare_exchange(-1, tid)) + __builtin_ia32_pause(); + m_flags = get_flags_and_disable_interrupts(); + } + m_lock_depth++; + } + + bool RecursiveSpinLock::try_lock() + { + auto tid = Scheduler::current_tid(); + if (m_locker != tid) + { + if (!m_locker.compare_exchange(-1, tid)) + return false; + m_flags = get_flags_and_disable_interrupts(); + } + m_lock_depth++; + return true; + } + + void RecursiveSpinLock::unlock() + { + ASSERT_EQ(m_locker.load(), Scheduler::current_tid()); + if (--m_lock_depth == 0) + { + restore_flags(m_flags); + m_locker = -1; + } + } + +} diff --git a/kernel/kernel/Lock/spin.o b/kernel/kernel/Lock/spin.o new file mode 100644 index 0000000000000000000000000000000000000000..ee7a6fc8aa9009555a8e0ed24ab214b7e9bc1f23 GIT binary patch literal 9896 zcmeHNeQ;A%7Jq39q*#1iP{=N_1np=$BOxgW6pJ)%9=t%oLW`n+X-rdE=|?7gsVG*f zDM)PNc4l>E-C1^=@gK(7*UdgJJ(mZJ{)?EKQunwUToa1_T7rE z_N?1KqKr$Y6=qzGZy$^9k_R8OsPU`4?`r;@>)qPJiH)OgqRF0X1&`n5KFi`3`^=Rn zjH|I>i#+shqHv=;c#A#p*Xsf5f?u@>JHTT(c;oh^e z!8o=o=jFi)d*Z>-p6dl&srcqk)31{x_NLE3BM;QD#+%Tv@y73k9yF3ZEc7yHy&AerBK^K4CsJ7^t8nn8?m7GpEHOBe&{E_qI2uuCTQ#HWSX4Z`#N;8!F}S92iIGA zlUi*>w^kENxjZK@T`CIXgBPJvEf413mr#d#t}T81R`r4dv^#vU)0XO< ziu;^VJF6lysHiCDnn=&o6=8R`d;rtla~u>)$LioA4_)=e-nNiqgNoFfjHZ2wXJBBN z@8#YXfV;->t(u6QOKpItQxN8$B#*aedK`pUu{@tPZi<%}B@FD8`Ol$OkMCUIWEyEY z7pB+Bi+cd%=w#j$PstnQKa@95t9{X=ny9d*BHv{)Z=&c4eN4IYFZu1D6MI8KUoyuV3q%sL>x!J?+Rv-oCrM@lZ;1Y}{Ku7Int<=3BIRYV1@I zTc(poj2g7o9%lYNZ_Rdti{s#8N_X*obkPegVjk-I$q(6^r}kCbY7e`Urb6bS$ z8628G+v5{>Lc#vvi@%>8Si^xUa_`5$);C)oe6nyvIWNhFzJhj5u*z1G2%T5lS0Mi^ zKh4B0I%20+O^)xCZ?7dHuGqENPuqvqfk5n9@iRAj6B}KL6$@bV^~meL^2Nq2oS)K;<;C`J@r}o0?%(4PkbbPrgRi4II4BW_^9UMS>j?*NXEt-32)J= zL(1sppkh04MRUX_AWWm4AsooJ7cRYn`HuIAM3oDbOZvdGEJ6t6*fn6jR0hW&$o`-H zw-H#lbGf@s+7ybkhgu!sKzmEj5zrj_yVvaRmg>SS?KK_2z0P1bEX|oME!^2AIi1Vk zzn1o3YqT-6Y%h~rR*Q1ZX7t=sSEp>N_is_QmbCd>K{ym~wsr&>OG@u{Xf2JUj*|V| zjwNfsLaC!E(9#-e)W=J5xy0&%&7sCn%J<>)F9LAFAQs&I`zGzM4 zukW7`o0&I`I=weByDNydEXxc2DKTqauWlH}u!DG>LtET87=t&KpVvtKAzu&i{R!`( zoKO9Q^~L4)UGF zoLH=qZ7MUgT8O`aOgtxWSS4HyQwcAtsDFa^**p-7Rf0q_&^T-J%$e-OLvk+Y;l$TW z0r;H@UqU$MC=v}%DmTHm67FE!lldGW+(kI|>}A3$3Fm9>&nEak32z{NXfnECf-i!h z;<&=Z$JQ#;s{rR?qAphKHo>1IJx7U|t!1dcPB;$?U%&4YUPKP^@#euMce{{3Y#oE1 z&4B0fr``l_Ho;>i_^T%PaTELl6Z|UR7|)~Z{wzssorC`i@^!q3mIhnffR_@Jwt9c$~xrb@sf*Tb@ z>B@?F!}r&_rzu#epTpcKw~HOf{+70It8e4BmTjBBywU~MTk2N&QN>gZ&YEiG2dP=j zhhAN#bfJIKaCdIpYr^j({!KLEDoW|gG;Z?xj7z%66Q*63dZ41G1-r3R> zGVwMS$5g*_a2Q6Ai{l4%tU#BiI9r~*i7bbUh)I{*EnH8g@Z|7fwHA4TMNVhram&GJ z7UCSN(}SHe=iSTo42LWa?FmKnJcR^CEY6E+KaL+q5Xi`XQirGU-%aNL91HAp=X!okIG*6Nr7{gk!to}sh#~sp zZNkwmeh%S!@O(l-9Qir^h6z4uf)~O5j)Z!C4nMAEKH=yO|5goEX^Fru+Fc>wqMmYt z9v3PB@)~eD79^jbXEE?`e;yKW(eB3vJ^bC^YZLy3I8Y!yu2Q`zl@e}@XN7<}fS3DI zWzcgs>Demqi+(l=xTxnbgC0^NbqM@oJf9VCQU6Nm~4mFG-Ac3jCs;$4vOc0{`ux zhtKaX1zgl~T)@SB`nczti{EP`cYJy)d!N*MSlnFjzf?qYk`THNA_xU{k zDepMW=aIh`a{L40vzzccOz;vD+-1NE$e+~)ocmL0f~zKYtqIP*Ct{&4xEf&VDAKQ-VyKY0Fd zJve=qu-0q5`UqXwM6zfTx&{{DU&+2Q{1 z_jk$J`9zx*coa%4qL+Sw^Nv#4|0t2n(q{D?OmU2CL6 zx1*Qnk=yA8iwxGAtFz5urv{B+M;m@NAh$xz{-#KvEhIVjL^`5;uJs82xaU7WjDOn_ zHB?fAe;+D>Hs0&;9)QG42h?#&=vaoDPPJ58#7j9~s7EBOvr+`1eun)Og(Q{Gy#(cO zZf(S8r}9oH(Y~0Q*tOMQ7cnGCAH4xjyw{=5A1l8SWZC#B|DegJf$a0|XdE|=-whoY z%Swa%#LsjIK2DUw7#qjm4;b1vh-WWxVZ(sAJcuxQ9-hoTwRvH~i2c@<3gbl&XtQ2qpWGc@z6L$H$T`mXz4I#?d9CJl^$2+m`4YA96-2jq?8m D--?JU literal 0 HcmV?d00001 diff --git a/kernel/kernel/Memory/FileBackedRegion.cpp b/kernel/kernel/Memory/FileBackedRegion.cpp index 22e5d649..09c527db 100644 --- a/kernel/kernel/Memory/FileBackedRegion.cpp +++ b/kernel/kernel/Memory/FileBackedRegion.cpp @@ -1,5 +1,4 @@ -#include -#include +#include #include #include @@ -26,7 +25,7 @@ namespace Kernel if (type == Type::SHARED) { - LockGuard _(inode->m_lock); + LockGuard _(inode->m_mutex); if (inode->m_shared_region.valid()) region->m_shared_data = inode->m_shared_region.lock(); else @@ -83,12 +82,9 @@ namespace Kernel if (pages[page_index] == 0) return; - { - CriticalScope _; - PageTable::with_fast_page(pages[page_index], [&] { - memcpy(page_buffer, PageTable::fast_page_as_ptr(), PAGE_SIZE); - }); - } + PageTable::with_fast_page(pages[page_index], [&] { + memcpy(page_buffer, PageTable::fast_page_as_ptr(), PAGE_SIZE); + }); if (auto ret = inode->write(page_index * PAGE_SIZE, BAN::ConstByteSpan::from(page_buffer)); ret.is_error()) dwarnln("{}", ret.error()); @@ -157,7 +153,7 @@ namespace Kernel } else if (m_type == Type::SHARED) { - LockGuard _(m_inode->m_lock); + LockGuard _(m_inode->m_mutex); ASSERT(m_inode->m_shared_region.valid()); ASSERT(m_shared_data->pages.size() == BAN::Math::div_round_up(m_inode->size(), PAGE_SIZE)); @@ -175,10 +171,9 @@ namespace Kernel TRY(m_inode->read(offset, BAN::ByteSpan(m_shared_data->page_buffer, bytes))); - CriticalScope _; - PageTable::map_fast_page(pages[page_index]); - memcpy(PageTable::fast_page_as_ptr(), m_shared_data->page_buffer, PAGE_SIZE); - PageTable::unmap_fast_page(); + PageTable::with_fast_page(pages[page_index], [&] { + memcpy(PageTable::fast_page_as_ptr(), m_shared_data->page_buffer, PAGE_SIZE); + }); } paddr_t paddr = pages[page_index]; diff --git a/kernel/kernel/Memory/Heap.cpp b/kernel/kernel/Memory/Heap.cpp index bc5d51d6..88faa814 100644 --- a/kernel/kernel/Memory/Heap.cpp +++ b/kernel/kernel/Memory/Heap.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include diff --git a/kernel/kernel/Memory/MemoryBackedRegion.cpp b/kernel/kernel/Memory/MemoryBackedRegion.cpp index 3b7066eb..fd506856 100644 --- a/kernel/kernel/Memory/MemoryBackedRegion.cpp +++ b/kernel/kernel/Memory/MemoryBackedRegion.cpp @@ -1,5 +1,4 @@ -#include -#include +#include #include #include @@ -60,12 +59,7 @@ namespace Kernel if (&PageTable::current() == &m_page_table) memset((void*)vaddr, 0x00, PAGE_SIZE); else - { - CriticalScope _; - PageTable::map_fast_page(paddr); - memset(PageTable::fast_page_as_ptr(), 0x00, PAGE_SIZE); - PageTable::unmap_fast_page(); - } + PageTable::with_fast_page(paddr, [] { memset(PageTable::fast_page_as_ptr(), 0x00, PAGE_SIZE); }); return true; } @@ -103,12 +97,10 @@ namespace Kernel if (&PageTable::current() == &m_page_table) memcpy((void*)write_vaddr, (void*)(buffer + written), bytes); else - { - CriticalScope _; - PageTable::map_fast_page(m_page_table.physical_address_of(write_vaddr & PAGE_ADDR_MASK)); - memcpy(PageTable::fast_page_as_ptr(page_offset), (void*)(buffer + written), bytes); - PageTable::unmap_fast_page(); - } + PageTable::with_fast_page( + m_page_table.physical_address_of(write_vaddr & PAGE_ADDR_MASK), + [&] { memcpy(PageTable::fast_page_as_ptr(page_offset), (void*)(buffer + written), bytes); } + ); written += bytes; } diff --git a/kernel/kernel/Memory/VirtualRange.cpp b/kernel/kernel/Memory/VirtualRange.cpp index 6aefe7db..6c8a7417 100644 --- a/kernel/kernel/Memory/VirtualRange.cpp +++ b/kernel/kernel/Memory/VirtualRange.cpp @@ -1,5 +1,4 @@ -#include -#include +#include #include #include @@ -135,10 +134,9 @@ namespace Kernel result->m_page_table.map_page_at(paddr, vaddr() + offset, m_flags); } - CriticalScope _; - PageTable::map_fast_page(result->m_page_table.physical_address_of(vaddr() + offset)); - memcpy(PageTable::fast_page_as_ptr(), (void*)(vaddr() + offset), PAGE_SIZE); - PageTable::unmap_fast_page(); + PageTable::with_fast_page(result->m_page_table.physical_address_of(vaddr() + offset), [&] { + memcpy(PageTable::fast_page_as_ptr(), (void*)(vaddr() + offset), PAGE_SIZE); + }); } return result; @@ -176,10 +174,9 @@ namespace Kernel for (size_t offset = 0; offset < size(); offset += PAGE_SIZE) { - CriticalScope _; - PageTable::map_fast_page(m_page_table.physical_address_of(vaddr() + offset)); - memset(PageTable::fast_page_as_ptr(), 0x00, PAGE_SIZE); - PageTable::unmap_fast_page(); + PageTable::with_fast_page(m_page_table.physical_address_of(vaddr() + offset), [&] { + memset(PageTable::fast_page_as_ptr(), 0x00, PAGE_SIZE); + }); } } @@ -204,12 +201,9 @@ namespace Kernel while (bytes > 0) { - { - CriticalScope _; - PageTable::map_fast_page(m_page_table.physical_address_of(vaddr() + page_index * PAGE_SIZE)); + PageTable::with_fast_page(m_page_table.physical_address_of(vaddr() + page_index * PAGE_SIZE), [&] { memcpy(PageTable::fast_page_as_ptr(page_offset), buffer, PAGE_SIZE - page_offset); - PageTable::unmap_fast_page(); - } + }); buffer += PAGE_SIZE - page_offset; bytes -= PAGE_SIZE - page_offset; diff --git a/kernel/kernel/Memory/kmalloc.cpp b/kernel/kernel/Memory/kmalloc.cpp index 4f1c5b0a..28e2208c 100644 --- a/kernel/kernel/Memory/kmalloc.cpp +++ b/kernel/kernel/Memory/kmalloc.cpp @@ -1,10 +1,13 @@ #include -#include #include +#include #include #include +using Kernel::LockGuard; +using Kernel::SpinLock; + #define MB (1 << 20) extern uint8_t g_kernel_end[]; @@ -81,6 +84,8 @@ struct kmalloc_info }; static kmalloc_info s_kmalloc_info; +static SpinLock s_kmalloc_lock; + template struct kmalloc_fixed_node { @@ -144,6 +149,8 @@ void kmalloc_initialize() void kmalloc_dump_info() { + LockGuard _(s_kmalloc_lock); + kprintln("kmalloc: 0x{8H}->0x{8H}", s_kmalloc_info.base, s_kmalloc_info.end); kprintln(" used: 0x{8H}", s_kmalloc_info.used); kprintln(" free: 0x{8H}", s_kmalloc_info.free); @@ -155,14 +162,18 @@ void kmalloc_dump_info() static bool is_corrupted() { + LockGuard _(s_kmalloc_lock); auto& info = s_kmalloc_info; auto* temp = info.first(); - for (; temp->end() <= info.end; temp = temp->after()); + for (; temp->end() <= info.end; temp = temp->after()) + continue; return (uintptr_t)temp != info.end; } [[maybe_unused]] static void debug_dump() { + LockGuard _(s_kmalloc_lock); + auto& info = s_kmalloc_info; uint32_t used = 0; @@ -183,6 +194,8 @@ static void* kmalloc_fixed() { auto& info = s_kmalloc_fixed_info; + LockGuard _(s_kmalloc_lock); + if (!info.free_list_head) return nullptr; @@ -225,6 +238,8 @@ static void* kmalloc_impl(size_t size, size_t align) auto& info = s_kmalloc_info; + LockGuard _(s_kmalloc_lock); + for (auto* node = info.first(); node->end() <= info.end; node = node->after()) { if (node->used()) @@ -304,8 +319,6 @@ void* kmalloc(size_t size, size_t align, bool force_identity_map) align = s_kmalloc_min_align; ASSERT(align <= PAGE_SIZE); - Kernel::CriticalScope critical; - if (size == 0 || size >= info.size) goto no_memory; @@ -338,7 +351,7 @@ void kfree(void* address) uintptr_t address_uint = (uintptr_t)address; ASSERT(address_uint % s_kmalloc_min_align == 0); - Kernel::CriticalScope critical; + LockGuard _(s_kmalloc_lock); if (s_kmalloc_fixed_info.base <= address_uint && address_uint < s_kmalloc_fixed_info.end) { @@ -399,10 +412,9 @@ void kfree(void* address) BAN::Optional kmalloc_paddr_of(Kernel::vaddr_t vaddr) { - using namespace Kernel; - + using Kernel::vaddr_t; + LockGuard _(s_kmalloc_lock); if ((vaddr_t)s_kmalloc_storage <= vaddr && vaddr < (vaddr_t)s_kmalloc_storage + sizeof(s_kmalloc_storage)) return V2P(vaddr); - return {}; } diff --git a/kernel/kernel/Networking/ARPTable.cpp b/kernel/kernel/Networking/ARPTable.cpp index 51e8387c..06b574cf 100644 --- a/kernel/kernel/Networking/ARPTable.cpp +++ b/kernel/kernel/Networking/ARPTable.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -52,7 +52,7 @@ namespace Kernel ipv4_address = interface.get_gateway(); { - LockGuard _(m_lock); + LockGuard _(m_table_lock); if (m_arp_table.contains(ipv4_address)) return m_arp_table[ipv4_address]; } @@ -74,7 +74,7 @@ namespace Kernel while (SystemTimer::get().ms_since_boot() < timeout) { { - LockGuard _(m_lock); + LockGuard _(m_table_lock); if (m_arp_table.contains(ipv4_address)) return m_arp_table[ipv4_address]; } @@ -114,7 +114,7 @@ namespace Kernel } case ARPOperation::Reply: { - LockGuard _(m_lock); + LockGuard _(m_table_lock); if (m_arp_table.contains(packet.spa)) { if (m_arp_table[packet.spa] != packet.sha) @@ -145,7 +145,7 @@ namespace Kernel BAN::Optional pending; { - CriticalScope _; + LockGuard _(m_pending_lock); if (!m_pending_packets.empty()) { pending = m_pending_packets.front(); @@ -168,12 +168,12 @@ namespace Kernel { auto& arp_packet = buffer.as(); + LockGuard _(m_pending_lock); if (m_pending_packets.full()) { dprintln("arp packet queue full"); return; } - m_pending_packets.push({ .interface = interface, .packet = arp_packet }); m_pending_semaphore.unblock(); } diff --git a/kernel/kernel/Networking/E1000/E1000.cpp b/kernel/kernel/Networking/E1000/E1000.cpp index 31f07f6b..9636f466 100644 --- a/kernel/kernel/Networking/E1000/E1000.cpp +++ b/kernel/kernel/Networking/E1000/E1000.cpp @@ -261,7 +261,7 @@ namespace Kernel { ASSERT_LTE(buffer.size() + sizeof(EthernetHeader), E1000_TX_BUFFER_SIZE); - CriticalScope _; + LockGuard _(m_lock); size_t tx_current = read32(REG_TDT) % E1000_TX_DESCRIPTOR_COUNT; diff --git a/kernel/kernel/Networking/IPv4Layer.cpp b/kernel/kernel/Networking/IPv4Layer.cpp index 42fca57a..4ede1dea 100644 --- a/kernel/kernel/Networking/IPv4Layer.cpp +++ b/kernel/kernel/Networking/IPv4Layer.cpp @@ -70,7 +70,7 @@ namespace Kernel void IPv4Layer::unbind_socket(BAN::RefPtr socket, uint16_t port) { - LockGuard _(m_lock); + LockGuard _(m_socket_lock); if (m_bound_sockets.contains(port)) { ASSERT(m_bound_sockets[port].valid()); @@ -88,11 +88,11 @@ namespace Kernel return BAN::Error::from_errno(EAFNOSUPPORT); auto& sockaddr_in = *reinterpret_cast(address); - LockGuard _(m_lock); + LockGuard _(m_socket_lock); uint16_t port = NetworkSocket::PORT_NONE; for (uint32_t i = 0; i < 100 && port == NetworkSocket::PORT_NONE; i++) - if (uint32_t temp = 0xC000 | (Random::get_u32() & 0x3FFF); !m_bound_sockets.contains(temp)) + if (uint32_t temp = 0xC000 | (Random::get_u32() & 0x3FFF); !m_bound_sockets.contains(temp) || !m_bound_sockets[temp].valid()) port = temp; for (uint32_t temp = 0xC000; temp < 0xFFFF && port == NetworkSocket::PORT_NONE; temp++) if (!m_bound_sockets.contains(temp)) @@ -124,11 +124,17 @@ namespace Kernel auto& sockaddr_in = *reinterpret_cast(address); uint16_t port = BAN::host_to_network_endian(sockaddr_in.sin_port); - LockGuard _(m_lock); + LockGuard _(m_socket_lock); - if (m_bound_sockets.contains(port)) - return BAN::Error::from_errno(EADDRINUSE); - TRY(m_bound_sockets.insert(port, TRY(socket->get_weak_ptr()))); + if (!m_bound_sockets.contains(port)) + TRY(m_bound_sockets.insert(port, TRY(socket->get_weak_ptr()))); + else + { + auto& bound = m_bound_sockets[port]; + if (bound.valid()) + return BAN::Error::from_errno(EADDRINUSE); + bound = TRY(socket->get_weak_ptr()); + } // FIXME: actually determine proper interface auto interface = NetworkManager::get().interfaces().front(); @@ -243,7 +249,7 @@ namespace Kernel BAN::RefPtr bound_socket; { - LockGuard _(m_lock); + LockGuard _(m_socket_lock); if (!m_bound_sockets.contains(dst_port)) { dprintln_if(DEBUG_IPV4, "no one is listening on port {}", dst_port); @@ -280,7 +286,7 @@ namespace Kernel BAN::Optional pending; { - CriticalScope _; + LockGuard _(m_packet_lock); if (!m_pending_packets.empty()) { pending = m_pending_packets.front(); @@ -300,7 +306,7 @@ namespace Kernel if (auto ret = handle_ipv4_packet(pending->interface, BAN::ByteSpan(buffer_start, ipv4_packet_size)); ret.is_error()) dwarnln("{}", ret.error()); - CriticalScope _; + LockGuard _(m_packet_lock); m_pending_total_size -= ipv4_packet_size; if (m_pending_total_size) memmove(buffer_start, buffer_start + ipv4_packet_size, m_pending_total_size); @@ -309,6 +315,8 @@ namespace Kernel void IPv4Layer::add_ipv4_packet(NetworkInterface& interface, BAN::ConstByteSpan buffer) { + LockGuard _(m_packet_lock); + if (m_pending_packets.full()) { dwarnln("IPv4 packet queue full"); diff --git a/kernel/kernel/Networking/TCPSocket.cpp b/kernel/kernel/Networking/TCPSocket.cpp index 7cda7baa..81245c44 100644 --- a/kernel/kernel/Networking/TCPSocket.cpp +++ b/kernel/kernel/Networking/TCPSocket.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -68,8 +68,6 @@ namespace Kernel void TCPSocket::on_close_impl() { - LockGuard _(m_lock); - if (!is_bound()) return; @@ -103,8 +101,6 @@ namespace Kernel if (address_len > (socklen_t)sizeof(sockaddr_storage)) address_len = sizeof(sockaddr_storage); - LockGuard _(m_lock); - ASSERT(!m_connection_info.has_value()); switch (m_state) @@ -139,7 +135,7 @@ namespace Kernel uint64_t wake_time_ms = SystemTimer::get().ms_since_boot() + 5000; while (m_state != State::Established) { - LockFreeGuard free(m_lock); + LockFreeGuard free(m_mutex); if (SystemTimer::get().ms_since_boot() >= wake_time_ms) return BAN::Error::from_errno(ECONNREFUSED); TRY(Thread::current().block_or_eintr_or_waketime(m_semaphore, wake_time_ms, true)); @@ -195,6 +191,8 @@ namespace Kernel void TCPSocket::add_protocol_header(BAN::ByteSpan packet, uint16_t dst_port, PseudoHeader pseudo_header) { + LockGuard _(m_mutex); + auto& header = packet.as(); memset(&header, 0, sizeof(TCPHeader)); memset(header.options, TCPOption::End, m_tcp_options_bytes); @@ -212,7 +210,6 @@ namespace Kernel { case State::Closed: { - LockGuard _(m_lock); header.syn = 1; add_tcp_header_option<0, TCPOption::MaximumSeqmentSize>(header, m_interface->payload_mtu() - m_network_layer.header_size()); add_tcp_header_option<4, TCPOption::WindowScale>(header, 0); @@ -233,7 +230,6 @@ namespace Kernel break; case State::CloseWait: { - LockGuard _(m_lock); header.ack = 1; header.fin = 1; m_state = State::LastAck; @@ -242,7 +238,6 @@ namespace Kernel } case State::FinWait1: { - LockGuard _(m_lock); header.ack = 1; header.fin = 1; m_state = State::FinWait2; @@ -250,7 +245,6 @@ namespace Kernel } case State::FinWait2: { - LockGuard _(m_lock); header.ack = 1; m_state = State::TimeWait; m_time_wait_start_ms = SystemTimer::get().ms_since_boot(); @@ -303,6 +297,10 @@ namespace Kernel auto payload = buffer.slice(header.data_offset * sizeof(uint32_t)); + // FIXME: Internet layer packet receive thread should not be able to be + // blocked by inode's mutex + LockGuard _(m_mutex); + switch (m_state) { case State::Closed: @@ -312,7 +310,6 @@ namespace Kernel if (!header.ack || !header.syn) break; - LockGuard _(m_lock); if (header.ack_number != m_send_window.current_seq) { @@ -345,8 +342,6 @@ namespace Kernel if (!header.ack) break; - LockGuard _(m_lock); - if (header.fin) { if (m_recv_window.start_seq + m_recv_window.data_size != header.seq_number) @@ -436,7 +431,7 @@ namespace Kernel set_connection_as_closed(); { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (m_should_ack) { @@ -518,7 +513,7 @@ namespace Kernel BAN::ErrorOr TCPSocket::recvfrom_impl(BAN::ByteSpan buffer, sockaddr*, socklen_t*) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (m_state == State::Closed) return BAN::Error::from_errno(ENOTCONN); @@ -542,7 +537,7 @@ namespace Kernel case State::Closing: ASSERT_NOT_REACHED(); }; - LockFreeGuard free(m_lock); + LockFreeGuard free(m_mutex); TRY(Thread::current().block_or_eintr_indefinite(m_semaphore)); } @@ -575,7 +570,7 @@ namespace Kernel return message.size(); } - LockGuard _(m_lock); + LockGuard _(m_mutex); if (m_state == State::Closed) return BAN::Error::from_errno(ENOTCONN); @@ -602,7 +597,7 @@ namespace Kernel if (m_send_window.data_size + message.size() <= m_send_window.buffer->size()) break; - LockFreeGuard free(m_lock); + LockFreeGuard free(m_mutex); TRY(Thread::current().block_or_eintr_indefinite(m_semaphore)); } @@ -634,7 +629,7 @@ namespace Kernel case State::Closing: ASSERT_NOT_REACHED(); }; - LockFreeGuard free(m_lock); + LockFreeGuard free(m_mutex); TRY(Thread::current().block_or_eintr_indefinite(m_semaphore)); } diff --git a/kernel/kernel/Networking/UDPSocket.cpp b/kernel/kernel/Networking/UDPSocket.cpp index f49e5bd9..75fd8115 100644 --- a/kernel/kernel/Networking/UDPSocket.cpp +++ b/kernel/kernel/Networking/UDPSocket.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/kernel/kernel/Networking/UNIX/Socket.cpp b/kernel/kernel/Networking/UNIX/Socket.cpp index 3f8eb1f4..51bf50a3 100644 --- a/kernel/kernel/Networking/UNIX/Socket.cpp +++ b/kernel/kernel/Networking/UNIX/Socket.cpp @@ -229,11 +229,11 @@ namespace Kernel BAN::ErrorOr UnixDomainSocket::add_packet(BAN::ConstByteSpan packet) { - LockGuard _(m_lock); + LockGuard _(m_mutex); while (m_packet_sizes.full() || m_packet_size_total + packet.size() > s_packet_buffer_size) { - LockFreeGuard _(m_lock); + LockFreeGuard _(m_mutex); TRY(Thread::current().block_or_eintr_indefinite(m_packet_semaphore)); } @@ -340,7 +340,7 @@ namespace Kernel while (m_packet_size_total == 0) { - LockFreeGuard _(m_lock); + LockFreeGuard _(m_mutex); TRY(Thread::current().block_or_eintr_indefinite(m_packet_semaphore)); } diff --git a/kernel/kernel/PIC.cpp b/kernel/kernel/PIC.cpp index c3ee85a8..547d8b9a 100644 --- a/kernel/kernel/PIC.cpp +++ b/kernel/kernel/PIC.cpp @@ -1,6 +1,6 @@ -#include #include #include +#include #include #include @@ -79,7 +79,7 @@ namespace Kernel void PIC::enable_irq(uint8_t irq) { - CriticalScope _; + LockGuard _(m_lock); ASSERT(irq < 16); ASSERT(m_reserved_irqs & (1 << irq)); @@ -99,7 +99,7 @@ namespace Kernel dwarnln("PIC only supports 16 irqs"); return BAN::Error::from_errno(EFAULT); } - CriticalScope _; + LockGuard _(m_lock); if (m_reserved_irqs & (1 << irq)) { dwarnln("irq {} is already reserved", irq); @@ -111,7 +111,7 @@ namespace Kernel BAN::Optional PIC::get_free_irq() { - CriticalScope _; + LockGuard _(m_lock); for (int irq = 0; irq < 16; irq++) { if (m_reserved_irqs & (1 << irq)) diff --git a/kernel/kernel/Process.cpp b/kernel/kernel/Process.cpp index e0f2e6c5..52114465 100644 --- a/kernel/kernel/Process.cpp +++ b/kernel/kernel/Process.cpp @@ -1,17 +1,15 @@ #include #include -#include #include #include #include #include #include #include -#include +#include #include #include #include -#include #include #include #include @@ -67,7 +65,7 @@ namespace Kernel pid_t pid; { - CriticalScope _; + LockGuard _(s_process_lock); pid = s_next_id; if (sid == 0 && pgrp == 0) { @@ -138,8 +136,6 @@ namespace Kernel char** argv = nullptr; { - PageTableScope _(process->page_table()); - size_t needed_bytes = sizeof(char*) * 2 + path.size() + 1; if (auto rem = needed_bytes % PAGE_SIZE) needed_bytes += PAGE_SIZE - rem; @@ -196,7 +192,7 @@ namespace Kernel void Process::add_thread(Thread* thread) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); MUST(m_threads.push_back(thread)); } @@ -217,7 +213,8 @@ namespace Kernel while (m_exit_status.waiting > 0) Scheduler::get().reschedule(); - m_lock.lock(); + // This mutex will no longer be freed + m_big_mutex.lock(); m_open_file_descriptors.close_all(); @@ -238,7 +235,7 @@ namespace Kernel m_threads.clear(); thread.setup_process_cleanup(); - Scheduler::get().execute_current_thread(); + Scheduler::get().reschedule_current_no_save(); ASSERT_NOT_REACHED(); } @@ -256,7 +253,7 @@ namespace Kernel void Process::exit(int status, int signal) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); m_exit_status.exit_code = __WGENEXITCODE(status, signal); for (auto* thread : m_threads) if (thread != &Thread::current()) @@ -277,7 +274,7 @@ namespace Kernel meminfo.phys_pages = 0; { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); for (auto* thread : m_threads) { meminfo.virt_pages += thread->virtual_page_count(); @@ -326,13 +323,13 @@ namespace Kernel size_t Process::proc_cmdline(off_t offset, BAN::ByteSpan buffer) const { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); return read_from_vec_of_str(m_cmdline, offset, buffer); } size_t Process::proc_environ(off_t offset, BAN::ByteSpan buffer) const { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); return read_from_vec_of_str(m_environ, offset, buffer); } @@ -345,7 +342,7 @@ namespace Kernel BAN::ErrorOr Process::sys_gettermios(::termios* termios) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(validate_pointer_access(termios, sizeof(::termios))); @@ -364,7 +361,7 @@ namespace Kernel BAN::ErrorOr Process::sys_settermios(const ::termios* termios) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(validate_pointer_access(termios, sizeof(::termios))); @@ -403,7 +400,7 @@ namespace Kernel { auto page_table = BAN::UniqPtr::adopt(TRY(PageTable::create_userspace())); - LockGuard _(m_lock); + LockGuard _(m_big_mutex); BAN::String working_directory; TRY(working_directory.append(m_working_directory)); @@ -443,7 +440,7 @@ namespace Kernel { // NOTE: We scope everything for automatic deletion { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(validate_string_access(path)); auto loadable_elf = TRY(load_elf_for_exec(m_credentials, path, m_working_directory, page_table())); @@ -542,7 +539,7 @@ namespace Kernel m_has_called_exec = true; m_threads.front()->setup_exec(); - Scheduler::get().execute_current_thread(); + Scheduler::get().reschedule_current_no_save(); ASSERT_NOT_REACHED(); } @@ -579,7 +576,7 @@ namespace Kernel BAN::ErrorOr Process::sys_wait(pid_t pid, int* stat_loc, int options) { { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(validate_pointer_access(stat_loc, sizeof(int))); } @@ -612,7 +609,7 @@ namespace Kernel BAN::ErrorOr Process::sys_nanosleep(const timespec* rqtp, timespec* rmtp) { { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(validate_pointer_access(rqtp, sizeof(timespec))); if (rmtp) TRY(validate_pointer_access(rmtp, sizeof(timespec))); @@ -654,7 +651,7 @@ namespace Kernel return BAN::Error::from_errno(ENOTSUP); } - LockGuard _(m_lock); + LockGuard _(m_big_mutex); auto absolute_path = TRY(absolute_path_of(path)); @@ -683,7 +680,7 @@ namespace Kernel { ASSERT(&Process::current() == this); - LockGuard _(m_lock); + LockGuard _(m_big_mutex); if (Thread::current().stack().contains(address)) { @@ -711,13 +708,13 @@ namespace Kernel BAN::ErrorOr Process::open_inode(BAN::RefPtr inode, int flags) { ASSERT(inode); - LockGuard _(m_lock); + LockGuard _(m_big_mutex); return TRY(m_open_file_descriptors.open(inode, flags)); } BAN::ErrorOr Process::open_file(BAN::StringView path, int flags, mode_t mode) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); BAN::String absolute_path = TRY(absolute_path_of(path)); @@ -750,14 +747,14 @@ namespace Kernel BAN::ErrorOr Process::sys_open(const char* path, int flags, mode_t mode) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(validate_string_access(path)); return open_file(path, flags, mode); } BAN::ErrorOr Process::sys_openat(int fd, const char* path, int flags, mode_t mode) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(validate_string_access(path)); @@ -773,28 +770,28 @@ namespace Kernel BAN::ErrorOr Process::sys_close(int fd) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(m_open_file_descriptors.close(fd)); return 0; } BAN::ErrorOr Process::sys_read(int fd, void* buffer, size_t count) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(validate_pointer_access(buffer, count)); return TRY(m_open_file_descriptors.read(fd, BAN::ByteSpan((uint8_t*)buffer, count))); } BAN::ErrorOr Process::sys_write(int fd, const void* buffer, size_t count) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(validate_pointer_access(buffer, count)); return TRY(m_open_file_descriptors.write(fd, BAN::ByteSpan((uint8_t*)buffer, count))); } BAN::ErrorOr Process::sys_create(const char* path, mode_t mode) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(validate_string_access(path)); TRY(create_file_or_dir(path, mode)); return 0; @@ -802,7 +799,7 @@ namespace Kernel BAN::ErrorOr Process::sys_create_dir(const char* path, mode_t mode) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(validate_string_access(path)); BAN::StringView path_sv(path); if (!path_sv.empty() && path_sv.back() == '/') @@ -813,7 +810,7 @@ namespace Kernel BAN::ErrorOr Process::sys_unlink(const char* path) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(validate_string_access(path)); auto absolute_path = TRY(absolute_path_of(path)); @@ -846,7 +843,7 @@ namespace Kernel BAN::ErrorOr Process::sys_readlink(const char* path, char* buffer, size_t bufsize) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(validate_string_access(path)); TRY(validate_pointer_access(buffer, bufsize)); @@ -857,7 +854,7 @@ namespace Kernel BAN::ErrorOr Process::sys_readlinkat(int fd, const char* path, char* buffer, size_t bufsize) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(validate_string_access(path)); TRY(validate_pointer_access(buffer, bufsize)); @@ -874,7 +871,7 @@ namespace Kernel BAN::ErrorOr Process::sys_pread(int fd, void* buffer, size_t count, off_t offset) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(validate_pointer_access(buffer, count)); auto inode = TRY(m_open_file_descriptors.inode_of(fd)); return TRY(inode->read(offset, { (uint8_t*)buffer, count })); @@ -885,7 +882,7 @@ namespace Kernel if (mode & S_IFMASK) return BAN::Error::from_errno(EINVAL); - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(validate_string_access(path)); auto absolute_path = TRY(absolute_path_of(path)); @@ -897,7 +894,7 @@ namespace Kernel BAN::ErrorOr Process::sys_chown(const char* path, uid_t uid, gid_t gid) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(validate_string_access(path)); auto absolute_path = TRY(absolute_path_of(path)); @@ -909,7 +906,7 @@ namespace Kernel BAN::ErrorOr Process::sys_socket(int domain, int type, int protocol) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); return TRY(m_open_file_descriptors.socket(domain, type, protocol)); } @@ -920,7 +917,7 @@ namespace Kernel if (!address && address_len) return BAN::Error::from_errno(EINVAL); - LockGuard _(m_lock); + LockGuard _(m_big_mutex); if (address) { TRY(validate_pointer_access(address_len, sizeof(*address_len))); @@ -936,7 +933,7 @@ namespace Kernel BAN::ErrorOr Process::sys_bind(int socket, const sockaddr* address, socklen_t address_len) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(validate_pointer_access(address, address_len)); auto inode = TRY(m_open_file_descriptors.inode_of(socket)); @@ -949,7 +946,7 @@ namespace Kernel BAN::ErrorOr Process::sys_connect(int socket, const sockaddr* address, socklen_t address_len) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(validate_pointer_access(address, address_len)); auto inode = TRY(m_open_file_descriptors.inode_of(socket)); @@ -962,7 +959,7 @@ namespace Kernel BAN::ErrorOr Process::sys_listen(int socket, int backlog) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); auto inode = TRY(m_open_file_descriptors.inode_of(socket)); if (!inode->mode().ifsock()) @@ -974,7 +971,7 @@ namespace Kernel BAN::ErrorOr Process::sys_sendto(const sys_sendto_t* arguments) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(validate_pointer_access(arguments, sizeof(sys_sendto_t))); TRY(validate_pointer_access(arguments->message, arguments->length)); TRY(validate_pointer_access(arguments->dest_addr, arguments->dest_len)); @@ -994,7 +991,7 @@ namespace Kernel if (!arguments->address && arguments->address_len) return BAN::Error::from_errno(EINVAL); - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(validate_pointer_access(arguments, sizeof(sys_recvfrom_t))); TRY(validate_pointer_access(arguments->buffer, arguments->length)); if (arguments->address) @@ -1013,14 +1010,14 @@ namespace Kernel BAN::ErrorOr Process::sys_ioctl(int fildes, int request, void* arg) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); auto inode = TRY(m_open_file_descriptors.inode_of(fildes)); return TRY(inode->ioctl(request, arg)); } BAN::ErrorOr Process::sys_pselect(sys_pselect_t* arguments) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(validate_pointer_access(arguments, sizeof(sys_pselect_t))); if (arguments->readfds) @@ -1089,7 +1086,7 @@ namespace Kernel if (set_bits > 0) break; - LockFreeGuard free(m_lock); + LockFreeGuard free(m_big_mutex); SystemTimer::get().sleep(1); } @@ -1115,7 +1112,7 @@ namespace Kernel BAN::ErrorOr Process::sys_pipe(int fildes[2]) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(validate_pointer_access(fildes, sizeof(int) * 2)); TRY(m_open_file_descriptors.pipe(fildes)); return 0; @@ -1123,32 +1120,32 @@ namespace Kernel BAN::ErrorOr Process::sys_dup(int fildes) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); return TRY(m_open_file_descriptors.dup(fildes)); } BAN::ErrorOr Process::sys_dup2(int fildes, int fildes2) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); return TRY(m_open_file_descriptors.dup2(fildes, fildes2)); } BAN::ErrorOr Process::sys_fcntl(int fildes, int cmd, int extra) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); return TRY(m_open_file_descriptors.fcntl(fildes, cmd, extra)); } BAN::ErrorOr Process::sys_seek(int fd, off_t offset, int whence) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(m_open_file_descriptors.seek(fd, offset, whence)); return 0; } BAN::ErrorOr Process::sys_tell(int fd) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); return TRY(m_open_file_descriptors.tell(fd)); } @@ -1156,7 +1153,7 @@ namespace Kernel { BAN::String absolute_source, absolute_target; { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(absolute_source.append(TRY(absolute_path_of(source)))); TRY(absolute_target.append(TRY(absolute_path_of(target)))); } @@ -1166,7 +1163,7 @@ namespace Kernel BAN::ErrorOr Process::sys_fstat(int fd, struct stat* buf) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(validate_pointer_access(buf, sizeof(struct stat))); TRY(m_open_file_descriptors.fstat(fd, buf)); return 0; @@ -1174,7 +1171,7 @@ namespace Kernel BAN::ErrorOr Process::sys_fstatat(int fd, const char* path, struct stat* buf, int flag) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(validate_pointer_access(buf, sizeof(struct stat))); TRY(m_open_file_descriptors.fstatat(fd, path, buf, flag)); return 0; @@ -1182,7 +1179,7 @@ namespace Kernel BAN::ErrorOr Process::sys_stat(const char* path, struct stat* buf, int flag) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(validate_pointer_access(buf, sizeof(struct stat))); TRY(m_open_file_descriptors.stat(TRY(absolute_path_of(path)), buf, flag)); return 0; @@ -1235,7 +1232,7 @@ namespace Kernel BAN::ErrorOr Process::sys_readdir(int fd, DirectoryEntryList* list, size_t list_size) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(validate_pointer_access(list, list_size)); TRY(m_open_file_descriptors.read_dir_entries(fd, list, list_size)); return 0; @@ -1246,7 +1243,7 @@ namespace Kernel BAN::String absolute_path; { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(validate_string_access(path)); absolute_path = TRY(absolute_path_of(path)); } @@ -1255,7 +1252,7 @@ namespace Kernel if (!file.inode->mode().ifdir()) return BAN::Error::from_errno(ENOTDIR); - LockGuard _(m_lock); + LockGuard _(m_big_mutex); m_working_directory = BAN::move(file.canonical_path); return 0; @@ -1263,7 +1260,7 @@ namespace Kernel BAN::ErrorOr Process::sys_getpwd(char* buffer, size_t size) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(validate_pointer_access(buffer, size)); @@ -1279,7 +1276,7 @@ namespace Kernel BAN::ErrorOr Process::sys_mmap(const sys_mmap_t* args) { { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(validate_pointer_access(args, sizeof(sys_mmap_t))); } @@ -1320,7 +1317,7 @@ namespace Kernel region_type, page_flags )); - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(m_mapped_regions.push_back(BAN::move(region))); return m_mapped_regions.back()->vaddr(); } @@ -1328,7 +1325,7 @@ namespace Kernel if (args->addr != nullptr) return BAN::Error::from_errno(ENOTSUP); - LockGuard _(m_lock); + LockGuard _(m_big_mutex); auto inode = TRY(m_open_file_descriptors.inode_of(args->fildes)); @@ -1376,7 +1373,7 @@ namespace Kernel if (vaddr % PAGE_SIZE != 0) return BAN::Error::from_errno(EINVAL); - LockGuard _(m_lock); + LockGuard _(m_big_mutex); // FIXME: We should only map partial regions for (size_t i = 0; i < m_mapped_regions.size(); i++) @@ -1395,7 +1392,7 @@ namespace Kernel if (vaddr % PAGE_SIZE != 0) return BAN::Error::from_errno(EINVAL); - LockGuard _(m_lock); + LockGuard _(m_big_mutex); for (auto& mapped_region : m_mapped_regions) if (mapped_region->overlaps(vaddr, len)) @@ -1406,7 +1403,7 @@ namespace Kernel BAN::ErrorOr Process::sys_tty_ctrl(int fildes, int command, int flags) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); auto inode = TRY(m_open_file_descriptors.inode_of(fildes)); if (!inode->is_tty()) @@ -1419,7 +1416,7 @@ namespace Kernel BAN::ErrorOr Process::sys_termid(char* buffer) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(validate_string_access(buffer)); @@ -1440,7 +1437,7 @@ namespace Kernel BAN::ErrorOr Process::sys_clock_gettime(clockid_t clock_id, timespec* tp) { { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(validate_pointer_access(tp, sizeof(timespec))); } @@ -1465,7 +1462,7 @@ namespace Kernel BAN::ErrorOr Process::sys_load_keymap(const char* path) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(validate_string_access(path)); if (!m_credentials.is_superuser()) @@ -1482,11 +1479,11 @@ namespace Kernel return BAN::Error::from_errno(EINVAL); { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); TRY(validate_pointer_access((void*)handler, sizeof(handler))); } - CriticalScope _; + LockGuard _(m_signal_lock); m_signal_handlers[signal] = (vaddr_t)handler; return 0; } @@ -1500,7 +1497,7 @@ namespace Kernel if (pid == Process::current().pid()) { - CriticalScope _; + LockGuard _(m_signal_lock); Process::current().m_signal_pending_mask |= 1 << signal; return 0; } @@ -1514,7 +1511,7 @@ namespace Kernel found = true; if (signal) { - CriticalScope _; + LockGuard _(m_signal_lock); process.m_signal_pending_mask |= 1 << signal; // FIXME: This is super hacky Scheduler::get().unblock_thread(process.m_threads.front()->tid()); @@ -1532,7 +1529,7 @@ namespace Kernel BAN::ErrorOr Process::sys_tcsetpgrp(int fd, pid_t pgrp) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); if (!m_controlling_terminal) return BAN::Error::from_errno(ENOTTY); @@ -1568,7 +1565,7 @@ namespace Kernel if (uid < 0 || uid >= 1'000'000'000) return BAN::Error::from_errno(EINVAL); - LockGuard _(m_lock); + LockGuard _(m_big_mutex); // If the process has appropriate privileges, setuid() shall set the real user ID, effective user ID, and the saved // set-user-ID of the calling process to uid. @@ -1598,7 +1595,7 @@ namespace Kernel if (gid < 0 || gid >= 1'000'000'000) return BAN::Error::from_errno(EINVAL); - LockGuard _(m_lock); + LockGuard _(m_big_mutex); // If the process has appropriate privileges, setgid() shall set the real group ID, effective group ID, and the saved // set-group-ID of the calling process to gid. @@ -1626,7 +1623,7 @@ namespace Kernel if (uid < 0 || uid >= 1'000'000'000) return BAN::Error::from_errno(EINVAL); - LockGuard _(m_lock); + LockGuard _(m_big_mutex); // If uid is equal to the real user ID or the saved set-user-ID, or if the process has appropriate privileges, seteuid() // shall set the effective user ID of the calling process to uid; the real user ID and saved set-user-ID shall remain unchanged. @@ -1645,7 +1642,7 @@ namespace Kernel if (gid < 0 || gid >= 1'000'000'000) return BAN::Error::from_errno(EINVAL); - LockGuard _(m_lock); + LockGuard _(m_big_mutex); // If gid is equal to the real group ID or the saved set-group-ID, or if the process has appropriate privileges, setegid() // shall set the effective group ID of the calling process to gid; the real group ID, saved set-group-ID, and any @@ -1673,7 +1670,7 @@ namespace Kernel // by the ruid and euid arguments. If ruid or euid is -1, the corresponding effective or real user ID of the current // process shall be left unchanged. - LockGuard _(m_lock); + LockGuard _(m_big_mutex); // A process with appropriate privileges can set either ID to any value. if (!m_credentials.is_superuser()) @@ -1721,7 +1718,7 @@ namespace Kernel // The real and effective group IDs may be set to different values in the same call. - LockGuard _(m_lock); + LockGuard _(m_big_mutex); // Only a process with appropriate privileges can set the real group ID and the effective group ID to any valid value. if (!m_credentials.is_superuser()) @@ -1754,7 +1751,7 @@ namespace Kernel if (pgid < 0) return BAN::Error::from_errno(EINVAL); - LockGuard _(m_lock); + LockGuard _(m_big_mutex); if (pid == 0) pid = m_pid; @@ -1819,7 +1816,7 @@ namespace Kernel BAN::ErrorOr Process::sys_getpgid(pid_t pid) { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); if (pid == 0 || pid == m_pid) return m_pgrp; @@ -1851,7 +1848,7 @@ namespace Kernel BAN::ErrorOr Process::absolute_path_of(BAN::StringView path) const { - LockGuard _(m_lock); + LockGuard _(m_big_mutex); if (path.empty() || path == "."sv) return m_working_directory; diff --git a/kernel/kernel/Scheduler.cpp b/kernel/kernel/Scheduler.cpp index 19f9039b..0c1d7c05 100644 --- a/kernel/kernel/Scheduler.cpp +++ b/kernel/kernel/Scheduler.cpp @@ -1,8 +1,8 @@ #include #include -#include #include #include +#include #include #include #include @@ -32,6 +32,39 @@ namespace Kernel asm volatile("movq %0, %%rsp" :: "r"(s_temp_stack + sizeof(s_temp_stack))); } + void SchedulerLock::lock() + { + auto tid = Scheduler::current_tid(); + if (tid != m_locker) + { + while (!m_locker.compare_exchange(-1, tid)) + __builtin_ia32_pause(); + ASSERT_EQ(m_lock_depth, 0); + } + m_lock_depth++; + } + + void SchedulerLock::unlock() + { + ASSERT_EQ(m_locker.load(), Scheduler::current_tid()); + ASSERT_GT(m_lock_depth, 0); + if (--m_lock_depth == 0) + m_locker = -1; + } + + void SchedulerLock::unlock_all() + { + ASSERT_EQ(m_locker.load(), Scheduler::current_tid()); + ASSERT_GT(m_lock_depth, 0); + m_lock_depth = 0; + m_locker = -1; + } + + pid_t SchedulerLock::locker() const + { + return m_locker; + } + BAN::ErrorOr Scheduler::initialize() { ASSERT(s_instance == nullptr); @@ -52,6 +85,7 @@ namespace Kernel VERIFY_CLI(); ASSERT(!m_active_threads.empty()); m_current_thread = m_active_threads.begin(); + m_lock.lock(); execute_current_thread(); ASSERT_NOT_REACHED(); } @@ -63,7 +97,7 @@ namespace Kernel pid_t Scheduler::current_tid() { - if (s_instance == nullptr) + if (s_instance == nullptr || s_instance->m_idle_thread == nullptr) return 0; return Scheduler::get().current_thread().tid(); } @@ -71,6 +105,7 @@ namespace Kernel void Scheduler::timer_reschedule() { VERIFY_CLI(); + m_lock.lock(); wake_threads(); @@ -84,6 +119,7 @@ namespace Kernel void Scheduler::reschedule() { DISABLE_INTERRUPTS(); + m_lock.lock(); if (save_current_thread()) { @@ -98,20 +134,30 @@ namespace Kernel void Scheduler::reschedule_if_idling() { VERIFY_CLI(); + m_lock.lock(); if (m_active_threads.empty() || ¤t_thread() != m_idle_thread) - return; + return m_lock.unlock(); if (save_current_thread()) return; - m_current_thread = m_active_threads.begin(); + m_current_thread = {}; + advance_current_thread(); execute_current_thread(); ASSERT_NOT_REACHED(); } + void Scheduler::reschedule_current_no_save() + { + VERIFY_CLI(); + m_lock.lock(); + execute_current_thread(); + } + void Scheduler::wake_threads() { VERIFY_CLI(); + ASSERT_EQ(m_lock.locker(), current_tid()); uint64_t current_time = SystemTimer::get().ms_since_boot(); while (!m_sleeping_threads.empty() && m_sleeping_threads.front().wake_time <= current_time) @@ -126,7 +172,7 @@ namespace Kernel BAN::ErrorOr Scheduler::add_thread(Thread* thread) { - CriticalScope _; + LockGuard _(m_lock); TRY(m_active_threads.emplace_back(thread)); return {}; } @@ -134,19 +180,20 @@ namespace Kernel void Scheduler::advance_current_thread() { VERIFY_CLI(); + ASSERT_EQ(m_lock.locker(), current_tid()); if (m_active_threads.empty()) - { m_current_thread = {}; - return; - } - if (!m_current_thread || ++m_current_thread == m_active_threads.end()) + else if (!m_current_thread || ++m_current_thread == m_active_threads.end()) m_current_thread = m_active_threads.begin(); + + m_lock.m_locker = current_tid(); } void Scheduler::remove_and_advance_current_thread() { VERIFY_CLI(); + ASSERT_EQ(m_lock.locker(), current_tid()); ASSERT(m_current_thread); @@ -161,6 +208,8 @@ namespace Kernel advance_current_thread(); m_active_threads.remove(temp); } + + m_lock.m_locker = current_tid(); } // NOTE: this is declared always inline, so we don't corrupt the stack @@ -168,6 +217,7 @@ namespace Kernel ALWAYS_INLINE bool Scheduler::save_current_thread() { VERIFY_CLI(); + ASSERT_EQ(m_lock.locker(), current_tid()); uintptr_t rsp, rip; push_callee_saved(); @@ -190,6 +240,7 @@ namespace Kernel void Scheduler::delete_current_process_and_thread() { DISABLE_INTERRUPTS(); + m_lock.lock(); load_temp_stack(); PageTable::kernel().load(); @@ -210,6 +261,7 @@ namespace Kernel void Scheduler::execute_current_thread() { VERIFY_CLI(); + ASSERT_EQ(m_lock.locker(), current_tid()); load_temp_stack(); PageTable::kernel().load(); @@ -220,6 +272,7 @@ namespace Kernel NEVER_INLINE void Scheduler::_execute_current_thread() { VERIFY_CLI(); + ASSERT_EQ(m_lock.locker(), current_tid()); #if SCHEDULER_VERIFY_STACK vaddr_t rsp; @@ -266,10 +319,12 @@ namespace Kernel { case Thread::State::NotStarted: current->set_started(); + m_lock.unlock_all(); start_thread(current->rsp(), current->rip()); case Thread::State::Executing: while (current->can_add_signal_to_execute()) current->handle_signal(); + m_lock.unlock_all(); continue_thread(current->rsp(), current->rip()); case Thread::State::Terminated: ASSERT_NOT_REACHED(); @@ -281,6 +336,7 @@ namespace Kernel void Scheduler::set_current_thread_sleeping_impl(uint64_t wake_time) { VERIFY_CLI(); + ASSERT_EQ(m_lock.locker(), current_tid()); if (save_current_thread()) { @@ -301,6 +357,7 @@ namespace Kernel ); m_current_thread = {}; + m_lock.m_locker = current_tid(); advance_current_thread(); execute_current_thread(); @@ -311,6 +368,7 @@ namespace Kernel { VERIFY_STI(); DISABLE_INTERRUPTS(); + m_lock.lock(); ASSERT(m_current_thread); @@ -322,6 +380,7 @@ namespace Kernel { VERIFY_STI(); DISABLE_INTERRUPTS(); + m_lock.lock(); ASSERT(m_current_thread); @@ -331,7 +390,7 @@ namespace Kernel void Scheduler::unblock_threads(Semaphore* semaphore) { - CriticalScope critical; + LockGuard _(m_lock); for (auto it = m_sleeping_threads.begin(); it != m_sleeping_threads.end();) { @@ -352,7 +411,7 @@ namespace Kernel void Scheduler::unblock_thread(pid_t tid) { - CriticalScope _; + LockGuard _(m_lock); for (auto it = m_sleeping_threads.begin(); it != m_sleeping_threads.end(); it++) { diff --git a/kernel/kernel/SpinLock.cpp b/kernel/kernel/SpinLock.cpp deleted file mode 100644 index 6a56a46f..00000000 --- a/kernel/kernel/SpinLock.cpp +++ /dev/null @@ -1,86 +0,0 @@ -#include -#include - -namespace Kernel -{ - - void SpinLock::lock() - { - pid_t tid = Scheduler::current_tid(); - ASSERT(tid != m_locker); - while (!m_locker.compare_exchange(-1, tid)) - Scheduler::get().reschedule(); - } - - void SpinLock::unlock() - { - ASSERT(m_locker == Scheduler::current_tid()); - m_locker = -1; - } - - bool SpinLock::is_locked() const - { - return m_locker != -1; - } - - void RecursiveSpinLock::lock() - { - pid_t tid = Scheduler::current_tid(); - if (m_locker != tid) - { - while (!m_locker.compare_exchange(-1, tid)) - Scheduler::get().reschedule(); - ASSERT(m_lock_depth == 0); - } - m_lock_depth++; - } - - void RecursiveSpinLock::unlock() - { - ASSERT(m_lock_depth > 0); - ASSERT(m_locker == Scheduler::current_tid()); - if (--m_lock_depth == 0) - m_locker = -1; - } - - bool RecursiveSpinLock::is_locked() const - { - return m_locker != -1; - } - - void RecursivePrioritySpinLock::lock() - { - pid_t tid = Scheduler::current_tid(); - - bool has_priority = !Thread::current().is_userspace(); - if (has_priority) - m_queue_length++; - - if (m_locker != tid) - { - while (!((has_priority || m_queue_length == 0) && m_locker.compare_exchange(-1, tid))) - Scheduler::get().reschedule(); - ASSERT(m_lock_depth == 0); - } - m_lock_depth++; - } - - void RecursivePrioritySpinLock::unlock() - { - ASSERT(m_lock_depth > 0); - ASSERT(m_locker == Scheduler::current_tid()); - - bool has_priority = !Thread::current().is_userspace(); - if (has_priority) - m_queue_length--; - - if (--m_lock_depth == 0) - m_locker = -1; - } - - bool RecursivePrioritySpinLock::is_locked() const - { - return m_locker != -1; - } - -} diff --git a/kernel/kernel/Storage/ATA/ATABus.cpp b/kernel/kernel/Storage/ATA/ATABus.cpp index 157ca480..7ff38820 100644 --- a/kernel/kernel/Storage/ATA/ATABus.cpp +++ b/kernel/kernel/Storage/ATA/ATABus.cpp @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include #include #include @@ -261,7 +261,7 @@ namespace Kernel if (lba + sector_count > device.sector_count()) return BAN::Error::from_error_code(ErrorCode::Storage_Boundaries); - LockGuard _(m_lock); + LockGuard _(m_mutex); if (lba < (1 << 28)) { @@ -298,7 +298,7 @@ namespace Kernel if (lba + sector_count > device.sector_count()) return BAN::Error::from_error_code(ErrorCode::Storage_Boundaries); - LockGuard _(m_lock); + LockGuard _(m_mutex); if (lba < (1 << 28)) { diff --git a/kernel/kernel/Storage/DiskCache.cpp b/kernel/kernel/Storage/DiskCache.cpp index 3e682b74..f72a4a1e 100644 --- a/kernel/kernel/Storage/DiskCache.cpp +++ b/kernel/kernel/Storage/DiskCache.cpp @@ -1,5 +1,4 @@ -#include -#include +#include #include #include #include @@ -32,10 +31,6 @@ namespace Kernel uint64_t page_cache_offset = sector % sectors_per_page; uint64_t page_cache_start = sector - page_cache_offset; - PageTable& page_table = PageTable::current(); - LockGuard page_table_locker(page_table); - ASSERT(page_table.is_page_free(0)); - for (auto& cache : m_cache) { if (cache.first_sector < page_cache_start) @@ -46,10 +41,9 @@ namespace Kernel if (!(cache.sector_mask & (1 << page_cache_offset))) continue; - CriticalScope _; - PageTable::map_fast_page(cache.paddr); - memcpy(buffer.data(), PageTable::fast_page_as_ptr(page_cache_offset * m_sector_size), m_sector_size); - PageTable::unmap_fast_page(); + PageTable::with_fast_page(cache.paddr, [&] { + memcpy(buffer.data(), PageTable::fast_page_as_ptr(page_cache_offset * m_sector_size), m_sector_size); + }); return true; } @@ -64,10 +58,6 @@ namespace Kernel uint64_t page_cache_offset = sector % sectors_per_page; uint64_t page_cache_start = sector - page_cache_offset; - PageTable& page_table = PageTable::current(); - LockGuard page_table_locker(page_table); - ASSERT(page_table.is_page_free(0)); - size_t index = 0; // Search the cache if the have this sector in memory @@ -80,12 +70,9 @@ namespace Kernel if (cache.first_sector > page_cache_start) break; - { - CriticalScope _; - PageTable::map_fast_page(cache.paddr); + PageTable::with_fast_page(cache.paddr, [&] { memcpy(PageTable::fast_page_as_ptr(page_cache_offset * m_sector_size), buffer.data(), m_sector_size); - PageTable::unmap_fast_page(); - } + }); cache.sector_mask |= 1 << page_cache_offset; if (dirty) @@ -111,12 +98,9 @@ namespace Kernel return ret.error(); } - { - CriticalScope _; - PageTable::map_fast_page(cache.paddr); + PageTable::with_fast_page(cache.paddr, [&] { memcpy(PageTable::fast_page_as_ptr(page_cache_offset * m_sector_size), buffer.data(), m_sector_size); - PageTable::unmap_fast_page(); - } + }); return {}; } @@ -128,12 +112,9 @@ namespace Kernel if (cache.dirty_mask == 0) continue; - { - CriticalScope _; - PageTable::map_fast_page(cache.paddr); + PageTable::with_fast_page(cache.paddr, [&] { memcpy(m_sync_cache.data(), PageTable::fast_page_as_ptr(), PAGE_SIZE); - PageTable::unmap_fast_page(); - } + }); uint8_t sector_start = 0; uint8_t sector_count = 0; diff --git a/kernel/kernel/Storage/NVMe/Queue.cpp b/kernel/kernel/Storage/NVMe/Queue.cpp index 1b0395e7..f841cc24 100644 --- a/kernel/kernel/Storage/NVMe/Queue.cpp +++ b/kernel/kernel/Storage/NVMe/Queue.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -44,7 +44,7 @@ namespace Kernel uint16_t NVMeQueue::submit_command(NVMe::SubmissionQueueEntry& sqe) { - LockGuard _(m_lock); + LockGuard _(m_mutex); ASSERT(m_done == false); m_status = 0; diff --git a/kernel/kernel/Storage/StorageDevice.cpp b/kernel/kernel/Storage/StorageDevice.cpp index 746403dc..46d320cd 100644 --- a/kernel/kernel/Storage/StorageDevice.cpp +++ b/kernel/kernel/Storage/StorageDevice.cpp @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include #include @@ -150,7 +150,7 @@ namespace Kernel BAN::Vector lba1; TRY(lba1.resize(sector_size())); - TRY(read_sectors(1, 1, lba1.span())); + TRY(read_sectors(1, 1, BAN::ByteSpan { lba1.span() })); const GPTHeader& header = *(const GPTHeader*)lba1.data(); if (!is_valid_gpt_header(header, sector_size())) @@ -165,7 +165,7 @@ namespace Kernel BAN::Vector entry_array; TRY(entry_array.resize(size)); - TRY(read_sectors(header.partition_entry_lba, size / sector_size(), entry_array.span())); + TRY(read_sectors(header.partition_entry_lba, size / sector_size(), BAN::ByteSpan { entry_array.span() })); if (!is_valid_gpt_crc32(header, lba1, entry_array)) return BAN::Error::from_error_code(ErrorCode::Storage_GPTHeader); @@ -207,14 +207,14 @@ namespace Kernel void StorageDevice::add_disk_cache() { - LockGuard _(m_lock); + LockGuard _(m_mutex); ASSERT(!m_disk_cache.has_value()); m_disk_cache.emplace(sector_size(), *this); } BAN::ErrorOr StorageDevice::sync_disk_cache() { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (m_disk_cache.has_value()) TRY(m_disk_cache->sync()); return {}; @@ -224,15 +224,12 @@ namespace Kernel { ASSERT(buffer.size() >= sector_count * sector_size()); - { - LockGuard _(m_lock); - if (!m_disk_cache.has_value()) - return read_sectors_impl(lba, sector_count, buffer); - } + LockGuard _(m_mutex); + if (!m_disk_cache.has_value()) + return read_sectors_impl(lba, sector_count, buffer); for (uint64_t offset = 0; offset < sector_count; offset++) { - LockGuard _(m_lock); auto sector_buffer = buffer.slice(offset * sector_size(), sector_size()); if (m_disk_cache->read_from_cache(lba + offset, sector_buffer)) continue; @@ -247,15 +244,12 @@ namespace Kernel { ASSERT(buffer.size() >= sector_count * sector_size()); - { - LockGuard _(m_lock); - if (!m_disk_cache.has_value()) - return write_sectors_impl(lba, sector_count, buffer); - } + LockGuard _(m_mutex); + if (!m_disk_cache.has_value()) + return write_sectors_impl(lba, sector_count, buffer); for (uint8_t offset = 0; offset < sector_count; offset++) { - LockGuard _(m_lock); auto sector_buffer = buffer.slice(offset * sector_size(), sector_size()); if (m_disk_cache->write_to_cache(lba + offset, sector_buffer, true).is_error()) TRY(write_sectors_impl(lba + offset, 1, sector_buffer)); diff --git a/kernel/kernel/Terminal/Serial.cpp b/kernel/kernel/Terminal/Serial.cpp index c7c64ba0..4035efe7 100644 --- a/kernel/kernel/Terminal/Serial.cpp +++ b/kernel/kernel/Terminal/Serial.cpp @@ -1,5 +1,4 @@ #include -#include #include #include #include @@ -235,7 +234,7 @@ namespace Kernel uint8_t buffer[128]; { - CriticalScope _; + LockGuard _(m_lock); if (m_input.empty()) return; uint8_t* ptr = buffer; diff --git a/kernel/kernel/Terminal/TTY.cpp b/kernel/kernel/Terminal/TTY.cpp index 14e1d09c..db79644b 100644 --- a/kernel/kernel/Terminal/TTY.cpp +++ b/kernel/kernel/Terminal/TTY.cpp @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include @@ -122,7 +122,7 @@ namespace Kernel void TTY::on_key_event(Input::KeyEvent event) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (event.released()) return; @@ -205,7 +205,7 @@ namespace Kernel if (ch == 0) return; - LockGuard _(m_lock); + LockGuard _(m_mutex); // ^C if (ch == '\x03') @@ -310,24 +310,18 @@ namespace Kernel void TTY::putchar(uint8_t ch) { + LockGuard _(m_mutex); if (m_tty_ctrl.draw_graphics) putchar_impl(ch); } BAN::ErrorOr TTY::read_impl(off_t, BAN::ByteSpan buffer) { - LockGuard _(m_lock); + LockGuard _(m_mutex); while (!m_output.flush) { - // FIXME: this is very hacky way to unlock lock temporarily - uint32_t depth = m_lock.lock_depth(); - for (uint32_t i = 0; i < depth; i++) - m_lock.unlock(); - auto eintr = Thread::current().block_or_eintr_indefinite(m_output.semaphore); - for (uint32_t i = 0; i < depth; i++) - m_lock.lock(); - if (eintr.is_error()) - return eintr.release_error(); + LockFreeGuard free(m_mutex); + TRY(Thread::current().block_or_eintr_indefinite(m_output.semaphore)); } if (m_output.bytes == 0) @@ -352,7 +346,7 @@ namespace Kernel BAN::ErrorOr TTY::write_impl(off_t, BAN::ConstByteSpan buffer) { - LockGuard _(m_lock); + LockGuard _(m_mutex); for (size_t i = 0; i < buffer.size(); i++) putchar(buffer[i]); return buffer.size(); @@ -361,7 +355,7 @@ namespace Kernel void TTY::putchar_current(uint8_t ch) { ASSERT(s_tty); - LockGuard _(s_tty->m_lock); + LockGuard _(s_tty->m_mutex); s_tty->putchar(ch); } diff --git a/kernel/kernel/Terminal/VirtualTTY.cpp b/kernel/kernel/Terminal/VirtualTTY.cpp index 59b5806c..d677a289 100644 --- a/kernel/kernel/Terminal/VirtualTTY.cpp +++ b/kernel/kernel/Terminal/VirtualTTY.cpp @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include @@ -57,6 +57,7 @@ namespace Kernel void VirtualTTY::clear() { + LockGuard _(m_write_lock); for (uint32_t i = 0; i < m_width * m_height; i++) m_buffer[i] = { .foreground = m_foreground, .background = m_background, .codepoint = ' ' }; m_terminal_driver->clear(m_background); @@ -64,6 +65,7 @@ namespace Kernel void VirtualTTY::set_font(const Kernel::Font& font) { + LockGuard _(m_write_lock); m_terminal_driver->set_font(font); uint32_t new_width = m_terminal_driver->width(); @@ -306,7 +308,7 @@ namespace Kernel void VirtualTTY::putchar_impl(uint8_t ch) { - ASSERT(m_lock.is_locked()); + LockGuard _(m_write_lock); uint32_t codepoint = ch; diff --git a/kernel/kernel/Thread.cpp b/kernel/kernel/Thread.cpp index 53c40086..b0dfd78c 100644 --- a/kernel/kernel/Thread.cpp +++ b/kernel/kernel/Thread.cpp @@ -3,8 +3,8 @@ #include #include #include +#include #include -#include #include #include #include @@ -30,10 +30,10 @@ namespace Kernel void Thread::terminate() { - CriticalScope _; + LockGuard _(m_lock); m_state = Thread::State::Terminated; if (this == &Thread::current()) - Scheduler::get().execute_current_thread(); + Scheduler::get().reschedule_current_no_save(); } static pid_t s_next_tid = 1; @@ -131,6 +131,8 @@ namespace Kernel BAN::ErrorOr Thread::clone(Process* new_process, uintptr_t rsp, uintptr_t rip) { + LockGuard _(m_lock); + ASSERT(m_is_userspace); ASSERT(m_state == State::Executing); @@ -156,6 +158,8 @@ namespace Kernel void Thread::setup_exec() { + LockGuard _(m_lock); + ASSERT(is_userspace()); m_state = State::NotStarted; static entry_t entry_trampoline( @@ -172,18 +176,24 @@ namespace Kernel // Signal mask is inherited // Setup stack for returning - { - // FIXME: don't use PageTableScope - PageTableScope _(process().page_table()); - write_to_stack(m_rsp, nullptr); // alignment - write_to_stack(m_rsp, this); - write_to_stack(m_rsp, &Thread::on_exit); - write_to_stack(m_rsp, nullptr); - } + uintptr_t offset = m_rsp % PAGE_SIZE; + if (offset == 0) + offset = PAGE_SIZE; + ASSERT_GTE(offset, 4 * sizeof(uintptr_t)); + PageTable::with_fast_page(process().page_table().physical_address_of((m_rsp - 4 * sizeof(uintptr_t)) & PAGE_ADDR_MASK), [&] { + uintptr_t rsp = PageTable::fast_page() + offset; + write_to_stack(rsp, nullptr); // alignment + write_to_stack(rsp, this); + write_to_stack(rsp, &Thread::on_exit); + write_to_stack(rsp, nullptr); + m_rsp -= 4 * sizeof(uintptr_t); + }); } void Thread::setup_process_cleanup() { + LockGuard _(m_lock); + m_state = State::NotStarted; static entry_t entry( [](void* process) @@ -199,19 +209,23 @@ namespace Kernel m_signal_pending_mask = 0; m_signal_block_mask = ~0ull; - // Setup stack for returning - { - // FIXME: don't use PageTableScope - PageTableScope _(process().page_table()); - write_to_stack(m_rsp, nullptr); // alignment - write_to_stack(m_rsp, this); - write_to_stack(m_rsp, &Thread::on_exit); - write_to_stack(m_rsp, m_process); - } + uintptr_t offset = m_rsp % PAGE_SIZE; + if (offset == 0) + offset = PAGE_SIZE; + ASSERT_GTE(offset, 4 * sizeof(uintptr_t)); + PageTable::with_fast_page(process().page_table().physical_address_of((m_rsp - 4 * sizeof(uintptr_t)) & PAGE_ADDR_MASK), [&] { + uintptr_t rsp = PageTable::fast_page() + offset; + write_to_stack(rsp, nullptr); // alignment + write_to_stack(rsp, this); + write_to_stack(rsp, &Thread::on_exit); + write_to_stack(rsp, m_process); + m_rsp -= 4 * sizeof(uintptr_t); + }); } bool Thread::is_interrupted_by_signal() { + LockGuard _(m_lock); while (can_add_signal_to_execute()) handle_signal(); return will_execute_signal(); @@ -219,6 +233,7 @@ namespace Kernel bool Thread::can_add_signal_to_execute() const { + LockGuard _(m_lock); if (!is_userspace() || m_state != State::Executing) return false; auto& interrupt_stack = *reinterpret_cast(interrupt_stack_base() + interrupt_stack_size() - sizeof(InterruptStack)); @@ -230,6 +245,7 @@ namespace Kernel bool Thread::will_execute_signal() const { + LockGuard _(m_lock); if (!is_userspace() || m_state != State::Executing) return false; auto& interrupt_stack = *reinterpret_cast(interrupt_stack_base() + interrupt_stack_size() - sizeof(InterruptStack)); @@ -238,6 +254,7 @@ namespace Kernel void Thread::handle_signal(int signal) { + LockGuard _(m_lock); ASSERT(!interrupts_enabled()); ASSERT(&Thread::current() == this); ASSERT(is_userspace()); @@ -331,6 +348,7 @@ namespace Kernel bool Thread::add_signal(int signal) { + LockGuard _(m_lock); ASSERT(!interrupts_enabled()); uint64_t mask = 1ull << signal; if (!(m_signal_block_mask & mask)) @@ -373,6 +391,7 @@ namespace Kernel void Thread::validate_stack() const { + LockGuard _(m_lock); if (stack_base() <= m_rsp && m_rsp <= stack_base() + stack_size()) return; if (interrupt_stack_base() <= m_rsp && m_rsp <= interrupt_stack_base() + interrupt_stack_size()) diff --git a/kernel/kernel/Timer/HPET.cpp b/kernel/kernel/Timer/HPET.cpp index f1c23814..ada4e3d8 100644 --- a/kernel/kernel/Timer/HPET.cpp +++ b/kernel/kernel/Timer/HPET.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -244,7 +245,7 @@ namespace Kernel if (m_is_64bit) return regs.main_counter.full; - CriticalScope _; + LockGuard _(m_lock); uint32_t current_low = regs.main_counter.low; uint32_t wraps = m_32bit_wraps; if (current_low < (uint32_t)m_last_ticks) @@ -256,19 +257,21 @@ namespace Kernel { auto& regs = registers(); - uint64_t current_ticks; - - if (m_is_64bit) - current_ticks = regs.main_counter.full; - else { - uint32_t current_low = regs.main_counter.low; - if (current_low < (uint32_t)m_last_ticks) - m_32bit_wraps++; - current_ticks = ((uint64_t)m_32bit_wraps << 32) | current_low; - } + LockGuard _(m_lock); - m_last_ticks = current_ticks; + uint64_t current_ticks; + if (m_is_64bit) + current_ticks = regs.main_counter.full; + else + { + uint32_t current_low = regs.main_counter.low; + if (current_low < (uint32_t)m_last_ticks) + m_32bit_wraps++; + current_ticks = ((uint64_t)m_32bit_wraps << 32) | current_low; + } + m_last_ticks = current_ticks; + } Scheduler::get().timer_reschedule(); }