diff --git a/LibELF/LibELF/ELF.cpp b/LibELF/LibELF/ELF.cpp index f1b4ebe4..566826cd 100644 --- a/LibELF/LibELF/ELF.cpp +++ b/LibELF/LibELF/ELF.cpp @@ -4,7 +4,6 @@ #ifdef __is_kernel #include -#include #include #endif diff --git a/LibELF/LibELF/LoadableELF.cpp b/LibELF/LibELF/LoadableELF.cpp index b8bccc17..e0a96cba 100644 --- a/LibELF/LibELF/LoadableELF.cpp +++ b/LibELF/LibELF/LoadableELF.cpp @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include #include @@ -282,10 +282,6 @@ namespace LibELF elf->reserve_address_space(); - ASSERT(&PageTable::current() == &m_page_table); - LockGuard _(m_page_table); - ASSERT(m_page_table.is_page_free(0)); - for (const auto& program_header : m_program_headers) { switch (program_header.p_type) @@ -315,12 +311,9 @@ namespace LibELF if (paddr == 0) return BAN::Error::from_errno(ENOMEM); - { - CriticalScope _; - PageTable::map_fast_page(paddr); + PageTable::with_fast_page(paddr, [&] { memcpy(PageTable::fast_page_as_ptr(), (void*)(start + i * PAGE_SIZE), PAGE_SIZE); - PageTable::unmap_fast_page(); - } + }); new_page_table.map_page_at(paddr, start + i * PAGE_SIZE, flags); elf->m_physical_page_count++; diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt index 30d904ad..5ff5f71c 100644 --- a/kernel/CMakeLists.txt +++ b/kernel/CMakeLists.txt @@ -42,6 +42,7 @@ set(KERNEL_SOURCES kernel/Input/PS2/Mouse.cpp kernel/InterruptController.cpp kernel/kernel.cpp + kernel/Lock/SpinLock.cpp kernel/Memory/DMARegion.cpp kernel/Memory/FileBackedRegion.cpp kernel/Memory/Heap.cpp @@ -68,7 +69,6 @@ set(KERNEL_SOURCES kernel/Random.cpp kernel/Scheduler.cpp kernel/Semaphore.cpp - kernel/SpinLock.cpp kernel/SSP.cpp kernel/Storage/ATA/AHCI/Controller.cpp kernel/Storage/ATA/AHCI/Device.cpp diff --git a/kernel/arch/x86_64/PageTable.cpp b/kernel/arch/x86_64/PageTable.cpp index f46c9078..e4ddbbd4 100644 --- a/kernel/arch/x86_64/PageTable.cpp +++ b/kernel/arch/x86_64/PageTable.cpp @@ -1,7 +1,7 @@ #include #include #include -#include +#include #include #include @@ -17,6 +17,8 @@ extern uint8_t g_userspace_end[]; namespace Kernel { + SpinLock PageTable::s_fast_page_lock; + static PageTable* s_kernel = nullptr; static PageTable* s_current = nullptr; static bool s_has_nxe = false; @@ -250,7 +252,7 @@ namespace Kernel BAN::ErrorOr PageTable::create_userspace() { - LockGuard _(s_kernel->m_lock); + SpinLockGuard _(s_kernel->m_lock); PageTable* page_table = new PageTable; if (page_table == nullptr) return BAN::Error::from_errno(ENOMEM); @@ -331,7 +333,7 @@ namespace Kernel uint64_t pde = (uc_vaddr >> 21) & 0x1FF; uint64_t pte = (uc_vaddr >> 12) & 0x1FF; - LockGuard _(m_lock); + SpinLockGuard _(m_lock); if (is_page_free(vaddr)) { @@ -353,7 +355,7 @@ namespace Kernel vaddr_t s_page = vaddr / PAGE_SIZE; vaddr_t e_page = BAN::Math::div_round_up(vaddr + size, PAGE_SIZE); - LockGuard _(m_lock); + SpinLockGuard _(m_lock); for (vaddr_t page = s_page; page < e_page; page++) unmap_page(page * PAGE_SIZE); } @@ -392,7 +394,7 @@ namespace Kernel // NOTE: we add present here, since it has to be available in higher level structures flags_t uwr_flags = (flags & (Flags::UserSupervisor | Flags::ReadWrite)) | Flags::Present; - LockGuard _(m_lock); + SpinLockGuard _(m_lock); uint64_t* pml4 = (uint64_t*)P2V(m_highest_paging_struct); if ((pml4[pml4e] & uwr_flags) != uwr_flags) @@ -437,7 +439,7 @@ namespace Kernel size_t page_count = range_page_count(vaddr, size); - LockGuard _(m_lock); + SpinLockGuard _(m_lock); for (size_t page = 0; page < page_count; page++) map_page_at(paddr + page * PAGE_SIZE, vaddr + page * PAGE_SIZE, flags); } @@ -454,7 +456,7 @@ namespace Kernel uint64_t pde = (uc_vaddr >> 21) & 0x1FF; uint64_t pte = (uc_vaddr >> 12) & 0x1FF; - LockGuard _(m_lock); + SpinLockGuard _(m_lock); uint64_t* pml4 = (uint64_t*)P2V(m_highest_paging_struct); if (!(pml4[pml4e] & Flags::Present)) @@ -488,7 +490,7 @@ namespace Kernel bool PageTable::reserve_page(vaddr_t vaddr, bool only_free) { - LockGuard _(m_lock); + SpinLockGuard _(m_lock); ASSERT(vaddr % PAGE_SIZE == 0); if (only_free && !is_page_free(vaddr)) return false; @@ -502,7 +504,7 @@ namespace Kernel bytes += PAGE_SIZE - rem; ASSERT(vaddr % PAGE_SIZE == 0); - LockGuard _(m_lock); + SpinLockGuard _(m_lock); if (only_free && !is_range_free(vaddr, bytes)) return false; for (size_t offset = 0; offset < bytes; offset += PAGE_SIZE) @@ -534,7 +536,7 @@ namespace Kernel const uint16_t e_pde = (uc_vaddr_end >> 21) & 0x1FF; const uint16_t e_pte = (uc_vaddr_end >> 12) & 0x1FF; - LockGuard _(m_lock); + SpinLockGuard _(m_lock); // Try to find free page that can be mapped without // allocations (page table with unused entries) @@ -607,7 +609,7 @@ namespace Kernel ASSERT(is_canonical(first_address)); ASSERT(is_canonical(last_address)); - LockGuard _(m_lock); + SpinLockGuard _(m_lock); for (vaddr_t vaddr = first_address; vaddr < last_address;) { @@ -648,7 +650,7 @@ namespace Kernel vaddr_t s_page = vaddr / PAGE_SIZE; vaddr_t e_page = BAN::Math::div_round_up(vaddr + size, PAGE_SIZE); - LockGuard _(m_lock); + SpinLockGuard _(m_lock); for (vaddr_t page = s_page; page < e_page; page++) if (!is_page_free(page * PAGE_SIZE)) return false; @@ -671,7 +673,7 @@ namespace Kernel void PageTable::debug_dump() { - LockGuard _(m_lock); + SpinLockGuard _(m_lock); flags_t flags = 0; vaddr_t start = 0; diff --git a/kernel/icxxabi.cpp b/kernel/icxxabi.cpp index 211b64a5..175318e1 100644 --- a/kernel/icxxabi.cpp +++ b/kernel/icxxabi.cpp @@ -1,6 +1,5 @@ -#include +#include #include -#include #define ATEXIT_MAX_FUNCS 128 @@ -115,22 +114,18 @@ void __cxa_finalize(void *f) namespace __cxxabiv1 { - /* guard variables */ - static Kernel::SpinLock s_spin_lock; - - /* The ABI requires a 64-bit type. */ - __extension__ typedef int __guard __attribute__((mode(__DI__))); + using __guard = uint64_t; int __cxa_guard_acquire (__guard* g) { - Kernel::LockGuard lock_guard(s_spin_lock); - return !*(int*)g; + auto& atomic = *reinterpret_cast*>(g); + return atomic == 0; } void __cxa_guard_release (__guard* g) { - Kernel::LockGuard lock_guard(s_spin_lock); - *(int*)g = 1; + auto& atomic = *reinterpret_cast*>(g); + atomic = 1; } void __cxa_guard_abort (__guard*) diff --git a/kernel/include/kernel/Debug.h b/kernel/include/kernel/Debug.h index 1e4ae987..59df777a 100644 --- a/kernel/include/kernel/Debug.h +++ b/kernel/include/kernel/Debug.h @@ -1,32 +1,30 @@ #pragma once #include +#include #define dprintln(...) \ do { \ - Debug::DebugLock::lock(); \ + Kernel::SpinLockGuard _(Debug::s_debug_lock); \ Debug::print_prefix(__FILE__, __LINE__); \ BAN::Formatter::print(Debug::putchar, __VA_ARGS__); \ BAN::Formatter::print(Debug::putchar, "\r\n"); \ - Debug::DebugLock::unlock(); \ } while(false) #define dwarnln(...) \ do { \ - Debug::DebugLock::lock(); \ + Kernel::SpinLockGuard _(Debug::s_debug_lock); \ BAN::Formatter::print(Debug::putchar, "\e[33m"); \ dprintln(__VA_ARGS__); \ BAN::Formatter::print(Debug::putchar, "\e[m"); \ - Debug::DebugLock::unlock(); \ } while(false) #define derrorln(...) \ do { \ - Debug::DebugLock::lock(); \ + Kernel::SpinLockGuard _(Debug::s_debug_lock); \ BAN::Formatter::print(Debug::putchar, "\e[31m"); \ dprintln(__VA_ARGS__); \ BAN::Formatter::print(Debug::putchar, "\e[m"); \ - Debug::DebugLock::unlock(); \ } while(false) #define dprintln_if(cond, ...) \ @@ -55,10 +53,5 @@ namespace Debug void putchar(char); void print_prefix(const char*, int); - class DebugLock - { - public: - static void lock(); - static void unlock(); - }; + extern Kernel::RecursiveSpinLock s_debug_lock; } diff --git a/kernel/include/kernel/FS/DevFS/FileSystem.h b/kernel/include/kernel/FS/DevFS/FileSystem.h index 333201e7..940da841 100644 --- a/kernel/include/kernel/FS/DevFS/FileSystem.h +++ b/kernel/include/kernel/FS/DevFS/FileSystem.h @@ -3,6 +3,7 @@ #include #include #include +#include #include namespace Kernel @@ -29,7 +30,7 @@ namespace Kernel { } private: - mutable SpinLock m_device_lock; + mutable Mutex m_device_lock; BAN::Vector> m_devices; diff --git a/kernel/include/kernel/FS/Ext2/FileSystem.h b/kernel/include/kernel/FS/Ext2/FileSystem.h index 95ab1978..8949aaf3 100644 --- a/kernel/include/kernel/FS/Ext2/FileSystem.h +++ b/kernel/include/kernel/FS/Ext2/FileSystem.h @@ -106,7 +106,7 @@ namespace Kernel }; private: - RecursiveSpinLock m_lock; + Mutex m_mutex; BAN::RefPtr m_block_device; diff --git a/kernel/include/kernel/FS/Inode.h b/kernel/include/kernel/FS/Inode.h index d5954d20..bd581680 100644 --- a/kernel/include/kernel/FS/Inode.h +++ b/kernel/include/kernel/FS/Inode.h @@ -9,7 +9,8 @@ #include #include -#include +#include +#include #include #include @@ -157,7 +158,7 @@ namespace Kernel virtual BAN::ErrorOr ioctl_impl(int request, void* arg) { return BAN::Error::from_errno(ENOTSUP); } protected: - mutable RecursivePrioritySpinLock m_lock; + mutable PriorityMutex m_mutex; private: BAN::WeakPtr m_shared_region; diff --git a/kernel/include/kernel/FS/Pipe.h b/kernel/include/kernel/FS/Pipe.h index ead80a3f..2d36dd72 100644 --- a/kernel/include/kernel/FS/Pipe.h +++ b/kernel/include/kernel/FS/Pipe.h @@ -2,7 +2,6 @@ #include #include -#include namespace Kernel { diff --git a/kernel/include/kernel/FS/TmpFS/FileSystem.h b/kernel/include/kernel/FS/TmpFS/FileSystem.h index 3a33dac8..4740d49b 100644 --- a/kernel/include/kernel/FS/TmpFS/FileSystem.h +++ b/kernel/include/kernel/FS/TmpFS/FileSystem.h @@ -4,9 +4,8 @@ #include #include #include -#include +#include #include -#include namespace Kernel { @@ -119,7 +118,7 @@ namespace Kernel private: const dev_t m_rdev; - RecursiveSpinLock m_lock; + Mutex m_mutex; BAN::HashMap> m_inode_cache; BAN::RefPtr m_root_inode; @@ -155,7 +154,7 @@ namespace Kernel template void TmpFileSystem::with_block_buffer(size_t index, F callback) { - LockGuard _(m_lock); + LockGuard _(m_mutex); paddr_t block_paddr = find_block(index); PageTable::with_fast_page(block_paddr, [&] { BAN::ByteSpan buffer(reinterpret_cast(PageTable::fast_page()), PAGE_SIZE); @@ -166,7 +165,7 @@ namespace Kernel template void TmpFileSystem::for_each_inode(F callback) { - LockGuard _(m_lock); + LockGuard _(m_mutex); for (auto& [_, inode] : m_inode_cache) { switch (callback(inode)) diff --git a/kernel/include/kernel/FS/VirtualFileSystem.h b/kernel/include/kernel/FS/VirtualFileSystem.h index 803caed9..1b56bf32 100644 --- a/kernel/include/kernel/FS/VirtualFileSystem.h +++ b/kernel/include/kernel/FS/VirtualFileSystem.h @@ -3,7 +3,7 @@ #include #include #include -#include +#include namespace Kernel { @@ -42,7 +42,7 @@ namespace Kernel MountPoint* mount_from_root_inode(BAN::RefPtr); private: - SpinLock m_lock; + Mutex m_mutex; FileSystem* m_root_fs = nullptr; BAN::Vector m_mount_points; }; diff --git a/kernel/include/kernel/Input/PS2/Controller.h b/kernel/include/kernel/Input/PS2/Controller.h index 16332da4..fc042e60 100644 --- a/kernel/include/kernel/Input/PS2/Controller.h +++ b/kernel/include/kernel/Input/PS2/Controller.h @@ -5,7 +5,7 @@ #include #include #include -#include +#include namespace Kernel::Input { @@ -60,7 +60,7 @@ namespace Kernel::Input private: BAN::RefPtr m_devices[2]; - RecursiveSpinLock m_lock; + Mutex m_mutex; BAN::CircularQueue m_command_queue; uint64_t m_command_send_time { 0 }; diff --git a/kernel/include/kernel/InterruptController.h b/kernel/include/kernel/InterruptController.h index 448d09b6..56dc6b25 100644 --- a/kernel/include/kernel/InterruptController.h +++ b/kernel/include/kernel/InterruptController.h @@ -51,6 +51,11 @@ namespace Kernel bool m_using_apic { false }; }; - bool interrupts_enabled(); + inline bool interrupts_enabled() + { + uintptr_t flags; + asm volatile("pushf; pop %0" : "=r"(flags) :: "memory"); + return flags & (1 << 9); + } } diff --git a/kernel/include/kernel/LockGuard.h b/kernel/include/kernel/Lock/LockGuard.h similarity index 100% rename from kernel/include/kernel/LockGuard.h rename to kernel/include/kernel/Lock/LockGuard.h diff --git a/kernel/include/kernel/Lock/Mutex.h b/kernel/include/kernel/Lock/Mutex.h new file mode 100644 index 00000000..27914ae6 --- /dev/null +++ b/kernel/include/kernel/Lock/Mutex.h @@ -0,0 +1,131 @@ +#pragma once + +#include +#include +#include + +#include + +namespace Kernel +{ + + class Mutex + { + BAN_NON_COPYABLE(Mutex); + BAN_NON_MOVABLE(Mutex); + + public: + Mutex() = default; + + void lock() + { + auto tid = Scheduler::current_tid(); + if (tid == m_locker) + ASSERT_GT(m_lock_depth, 0); + else + { + while (!m_locker.compare_exchange(-1, tid)) + Scheduler::get().reschedule(); + ASSERT_EQ(m_lock_depth, 0); + } + m_lock_depth++; + } + + bool try_lock() + { + auto tid = Scheduler::current_tid(); + if (tid == m_locker) + ASSERT_GT(m_lock_depth, 0); + else + { + if (!m_locker.compare_exchange(-1, tid)) + return false; + ASSERT_EQ(m_lock_depth, 0); + } + m_lock_depth++; + } + + void unlock() + { + ASSERT_EQ(m_locker.load(), Scheduler::current_tid()); + ASSERT_GT(m_lock_depth, 0); + if (--m_lock_depth == 0) + m_locker = -1; + } + + pid_t locker() const { return m_locker; } + bool is_locked() const { return m_locker != -1; } + uint32_t lock_depth() const { return m_lock_depth; } + + private: + BAN::Atomic m_locker { -1 }; + uint32_t m_lock_depth { 0 }; + }; + + class PriorityMutex + { + BAN_NON_COPYABLE(PriorityMutex); + BAN_NON_MOVABLE(PriorityMutex); + + public: + PriorityMutex() = default; + + void lock() + { + auto tid = Scheduler::current_tid(); + if (tid == m_locker) + ASSERT_GT(m_lock_depth, 0); + else + { + bool has_priority = tid ? !Thread::current().is_userspace() : true; + if (has_priority) + m_queue_length++; + while (!(has_priority || m_queue_length == 0) || !m_locker.compare_exchange(-1, tid)) + Scheduler::get().reschedule(); + ASSERT_EQ(m_lock_depth, 0); + } + m_lock_depth++; + } + + bool try_lock() + { + auto tid = Scheduler::current_tid(); + if (tid == m_locker) + ASSERT_GT(m_lock_depth, 0); + else + { + bool has_priority = tid ? !Thread::current().is_userspace() : true; + if (!(has_priority || m_queue_length == 0) || !m_locker.compare_exchange(-1, tid)) + return false; + if (has_priority) + m_queue_length++; + ASSERT_EQ(m_lock_depth, 0); + } + m_lock_depth++; + } + + void unlock() + { + auto tid = Scheduler::current_tid(); + ASSERT_EQ(m_locker.load(), tid); + ASSERT_GT(m_lock_depth, 0); + if (--m_lock_depth == 0) + { + bool has_priority = tid ? !Thread::current().is_userspace() : true; + if (has_priority) + m_queue_length--; + m_locker = -1; + } + } + + pid_t locker() const { return m_locker; } + bool is_locked() const { return m_locker != -1; } + uint32_t lock_depth() const { return m_lock_depth; } + + private: + BAN::Atomic m_locker { -1 }; + uint32_t m_lock_depth { 0 }; + BAN::Atomic m_queue_length { 0 }; + }; + +} diff --git a/kernel/include/kernel/Lock/SpinLock.h b/kernel/include/kernel/Lock/SpinLock.h new file mode 100644 index 00000000..9606f763 --- /dev/null +++ b/kernel/include/kernel/Lock/SpinLock.h @@ -0,0 +1,67 @@ +#pragma once + +#include +#include + +#include + +namespace Kernel +{ + + using InterruptState = bool; + + class SpinLock + { + BAN_NON_COPYABLE(SpinLock); + BAN_NON_MOVABLE(SpinLock); + + public: + SpinLock() = default; + + InterruptState lock(); + void unlock(InterruptState state); + + private: + BAN::Atomic m_locker { -1 }; + }; + + class RecursiveSpinLock + { + BAN_NON_COPYABLE(RecursiveSpinLock); + BAN_NON_MOVABLE(RecursiveSpinLock); + + public: + RecursiveSpinLock() = default; + + InterruptState lock(); + void unlock(InterruptState state); + + private: + BAN::Atomic m_locker { -1 }; + uint32_t m_lock_depth { 0 }; + }; + + template + class SpinLockGuard + { + BAN_NON_COPYABLE(SpinLockGuard); + BAN_NON_MOVABLE(SpinLockGuard); + + public: + SpinLockGuard(Lock& lock) + : m_lock(lock) + { + m_state = m_lock.lock(); + } + + ~SpinLockGuard() + { + m_lock.unlock(m_state); + } + + private: + Lock& m_lock; + InterruptState m_state; + }; + +} diff --git a/kernel/include/kernel/Memory/Heap.h b/kernel/include/kernel/Memory/Heap.h index 219f7b8c..97fe3457 100644 --- a/kernel/include/kernel/Memory/Heap.h +++ b/kernel/include/kernel/Memory/Heap.h @@ -3,8 +3,8 @@ #include #include +#include #include -#include namespace Kernel { diff --git a/kernel/include/kernel/Memory/PageTable.h b/kernel/include/kernel/Memory/PageTable.h index ccd3d2ab..e6190fb5 100644 --- a/kernel/include/kernel/Memory/PageTable.h +++ b/kernel/include/kernel/Memory/PageTable.h @@ -3,8 +3,8 @@ #include #include #include +#include #include -#include namespace Kernel { @@ -15,6 +15,12 @@ namespace Kernel requires BAN::is_same_v; }; + template + concept with_fast_page_callback_error = requires(F func) + { + requires BAN::is_same_v>; + }; + class PageTable { public: @@ -37,19 +43,27 @@ namespace Kernel static PageTable& kernel(); static PageTable& current(); - static void map_fast_page(paddr_t); - static void unmap_fast_page(); static constexpr vaddr_t fast_page() { return KERNEL_OFFSET; } template static void with_fast_page(paddr_t paddr, F callback) { - CriticalScope _; + SpinLockGuard _(s_fast_page_lock); map_fast_page(paddr); callback(); unmap_fast_page(); } + template + static BAN::ErrorOr with_fast_page(paddr_t paddr, F callback) + { + SpinLockGuard _(s_fast_page_lock); + map_fast_page(paddr); + auto ret = callback(); + unmap_fast_page(); + return ret; + } + // FIXME: implement sized checks, return span, etc static void* fast_page_as_ptr(size_t offset = 0) { @@ -97,8 +111,8 @@ namespace Kernel void load(); - void lock() const { m_lock.lock(); } - void unlock() const { m_lock.unlock(); } + InterruptState lock() const { return m_lock.lock(); } + void unlock(InterruptState state) const { m_lock.unlock(state); } void debug_dump(); @@ -110,9 +124,13 @@ namespace Kernel void prepare_fast_page(); static void invalidate(vaddr_t); + static void map_fast_page(paddr_t); + static void unmap_fast_page(); + private: paddr_t m_highest_paging_struct { 0 }; mutable RecursiveSpinLock m_lock; + static SpinLock s_fast_page_lock; }; static constexpr size_t range_page_count(vaddr_t start, size_t bytes) diff --git a/kernel/include/kernel/Memory/PageTableScope.h b/kernel/include/kernel/Memory/PageTableScope.h deleted file mode 100644 index 90eb74c4..00000000 --- a/kernel/include/kernel/Memory/PageTableScope.h +++ /dev/null @@ -1,33 +0,0 @@ -#pragma once - -#include -#include -#include - -namespace Kernel -{ - - class PageTableScope - { - public: - PageTableScope(PageTable& page_table) - : m_guard(page_table) - , m_old(PageTable::current()) - , m_temp(page_table) - { - if (&m_old != &m_temp) - m_temp.load(); - } - ~PageTableScope() - { - if (&m_old != &m_temp) - m_old.load(); - } - private: - LockGuard m_guard; - CriticalScope m_scope; - PageTable& m_old; - PageTable& m_temp; - }; - -} diff --git a/kernel/include/kernel/Networking/ARPTable.h b/kernel/include/kernel/Networking/ARPTable.h index ab4e99a6..3a24d290 100644 --- a/kernel/include/kernel/Networking/ARPTable.h +++ b/kernel/include/kernel/Networking/ARPTable.h @@ -51,7 +51,7 @@ namespace Kernel }; private: - SpinLock m_lock; + SpinLock m_table_lock; BAN::HashMap m_arp_table; diff --git a/kernel/include/kernel/Networking/IPv4Layer.h b/kernel/include/kernel/Networking/IPv4Layer.h index 7c599132..f7136e4b 100644 --- a/kernel/include/kernel/Networking/IPv4Layer.h +++ b/kernel/include/kernel/Networking/IPv4Layer.h @@ -12,7 +12,6 @@ #include #include #include -#include namespace Kernel { @@ -68,7 +67,7 @@ namespace Kernel }; private: - RecursiveSpinLock m_lock; + RecursiveSpinLock m_bound_socket_lock; BAN::UniqPtr m_arp_table; Process* m_process { nullptr }; diff --git a/kernel/include/kernel/Networking/TCPSocket.h b/kernel/include/kernel/Networking/TCPSocket.h index d82b4b38..839142a7 100644 --- a/kernel/include/kernel/Networking/TCPSocket.h +++ b/kernel/include/kernel/Networking/TCPSocket.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -119,8 +120,8 @@ namespace Kernel uint64_t m_time_wait_start_ms { 0 }; - RecursiveSpinLock m_lock; - Semaphore m_semaphore; + Mutex m_lock; + Semaphore m_semaphore; BAN::Atomic m_should_ack { false }; diff --git a/kernel/include/kernel/Networking/UDPSocket.h b/kernel/include/kernel/Networking/UDPSocket.h index d9bdae9b..ae1b48a1 100644 --- a/kernel/include/kernel/Networking/UDPSocket.h +++ b/kernel/include/kernel/Networking/UDPSocket.h @@ -2,6 +2,7 @@ #include #include +#include #include #include #include diff --git a/kernel/include/kernel/Networking/UNIX/Socket.h b/kernel/include/kernel/Networking/UNIX/Socket.h index b58ab646..4e46bd84 100644 --- a/kernel/include/kernel/Networking/UNIX/Socket.h +++ b/kernel/include/kernel/Networking/UNIX/Socket.h @@ -4,6 +4,7 @@ #include #include #include +#include namespace Kernel { @@ -65,6 +66,7 @@ namespace Kernel BAN::CircularQueue m_packet_sizes; size_t m_packet_size_total { 0 }; BAN::UniqPtr m_packet_buffer; + SpinLock m_packet_lock; Semaphore m_packet_semaphore; friend class BAN::RefPtr; diff --git a/kernel/include/kernel/Process.h b/kernel/include/kernel/Process.h index ad757159..ec9ccd54 100644 --- a/kernel/include/kernel/Process.h +++ b/kernel/include/kernel/Process.h @@ -7,10 +7,10 @@ #include #include #include +#include #include #include #include -#include #include #include @@ -51,9 +51,6 @@ namespace Kernel void register_to_scheduler(); void exit(int status, int signal); - static void for_each_process(const BAN::Function& callback); - static void for_each_process_in_session(pid_t sid, const BAN::Function& callback); - void add_thread(Thread*); void on_thread_exit(Thread&); @@ -222,7 +219,7 @@ namespace Kernel const pid_t m_pid; const pid_t m_parent; - mutable RecursiveSpinLock m_lock; + mutable Mutex m_process_lock; BAN::String m_working_directory; BAN::Vector m_threads; diff --git a/kernel/include/kernel/SpinLock.h b/kernel/include/kernel/SpinLock.h deleted file mode 100644 index 4585d3f3..00000000 --- a/kernel/include/kernel/SpinLock.h +++ /dev/null @@ -1,65 +0,0 @@ -#pragma once - -#include -#include - -#include - -namespace Kernel -{ - - class SpinLock - { - BAN_NON_COPYABLE(SpinLock); - BAN_NON_MOVABLE(SpinLock); - - public: - SpinLock() = default; - void lock(); - void unlock(); - bool is_locked() const; - - uint32_t lock_depth() const { return m_locker != -1; } - - private: - BAN::Atomic m_locker = -1; - }; - - class RecursiveSpinLock - { - BAN_NON_COPYABLE(RecursiveSpinLock); - BAN_NON_MOVABLE(RecursiveSpinLock); - - public: - RecursiveSpinLock() = default; - void lock(); - void unlock(); - bool is_locked() const; - - uint32_t lock_depth() const { return m_lock_depth; } - - private: - BAN::Atomic m_locker = -1; - BAN::Atomic m_lock_depth = 0; - }; - - class RecursivePrioritySpinLock - { - BAN_NON_COPYABLE(RecursivePrioritySpinLock); - BAN_NON_MOVABLE(RecursivePrioritySpinLock); - - public: - RecursivePrioritySpinLock() = default; - void lock(); - void unlock(); - bool is_locked() const; - - uint32_t lock_depth() const { return m_lock_depth; } - - private: - BAN::Atomic m_locker = -1; - BAN::Atomic m_lock_depth = 0; - BAN::Atomic m_queue_length = 0; - }; - -} diff --git a/kernel/include/kernel/Storage/ATA/ATABus.h b/kernel/include/kernel/Storage/ATA/ATABus.h index f192613a..16991266 100644 --- a/kernel/include/kernel/Storage/ATA/ATABus.h +++ b/kernel/include/kernel/Storage/ATA/ATABus.h @@ -4,7 +4,7 @@ #include #include #include -#include +#include namespace Kernel { @@ -51,7 +51,7 @@ namespace Kernel private: const uint16_t m_base; const uint16_t m_ctrl; - SpinLock m_lock; + Mutex m_mutex; volatile bool m_has_got_irq { false }; diff --git a/kernel/include/kernel/Storage/DiskCache.h b/kernel/include/kernel/Storage/DiskCache.h index 006a113f..a5a79313 100644 --- a/kernel/include/kernel/Storage/DiskCache.h +++ b/kernel/include/kernel/Storage/DiskCache.h @@ -3,7 +3,6 @@ #include #include #include -#include namespace Kernel { diff --git a/kernel/include/kernel/Storage/NVMe/Queue.h b/kernel/include/kernel/Storage/NVMe/Queue.h index 357a3a0a..964cf616 100644 --- a/kernel/include/kernel/Storage/NVMe/Queue.h +++ b/kernel/include/kernel/Storage/NVMe/Queue.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -20,7 +21,7 @@ namespace Kernel virtual void handle_irq() final override; private: - SpinLock m_lock; + Mutex m_mutex; BAN::UniqPtr m_completion_queue; BAN::UniqPtr m_submission_queue; volatile NVMe::DoorbellRegisters& m_doorbell; diff --git a/kernel/include/kernel/Storage/StorageDevice.h b/kernel/include/kernel/Storage/StorageDevice.h index aa8dc7f4..37599678 100644 --- a/kernel/include/kernel/Storage/StorageDevice.h +++ b/kernel/include/kernel/Storage/StorageDevice.h @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -44,7 +45,7 @@ namespace Kernel virtual bool has_error_impl() const override { return false; } private: - SpinLock m_lock; + Mutex m_mutex; BAN::Optional m_disk_cache; BAN::Vector> m_partitions; diff --git a/kernel/include/kernel/Terminal/TTY.h b/kernel/include/kernel/Terminal/TTY.h index e88b2915..f4499bc4 100644 --- a/kernel/include/kernel/Terminal/TTY.h +++ b/kernel/include/kernel/Terminal/TTY.h @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include #include @@ -84,6 +84,8 @@ namespace Kernel Semaphore semaphore; }; Buffer m_output; + + RecursiveSpinLock m_write_lock; }; } diff --git a/kernel/include/kernel/Terminal/VirtualTTY.h b/kernel/include/kernel/Terminal/VirtualTTY.h index e8b7a117..70ba61ad 100644 --- a/kernel/include/kernel/Terminal/VirtualTTY.h +++ b/kernel/include/kernel/Terminal/VirtualTTY.h @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include diff --git a/kernel/kernel/ACPI.cpp b/kernel/kernel/ACPI.cpp index ff397d97..fabd8e38 100644 --- a/kernel/kernel/ACPI.cpp +++ b/kernel/kernel/ACPI.cpp @@ -99,33 +99,39 @@ namespace Kernel if (rsdp->revision >= 2) { - PageTable::map_fast_page(rsdp->xsdt_address & PAGE_ADDR_MASK); - auto& xsdt = PageTable::fast_page_as(rsdp->xsdt_address % PAGE_SIZE); - BAN::ScopeGuard _([] { PageTable::unmap_fast_page(); }); + TRY(PageTable::with_fast_page(rsdp->xsdt_address & PAGE_ADDR_MASK, + [&]() -> BAN::ErrorOr + { + auto& xsdt = PageTable::fast_page_as(rsdp->xsdt_address % PAGE_SIZE); + if (memcmp(xsdt.signature, "XSDT", 4) != 0) + return BAN::Error::from_error_code(ErrorCode::ACPI_RootInvalid); + if (!is_valid_std_header(&xsdt)) + return BAN::Error::from_error_code(ErrorCode::ACPI_RootInvalid); - if (memcmp(xsdt.signature, "XSDT", 4) != 0) - return BAN::Error::from_error_code(ErrorCode::ACPI_RootInvalid); - if (!is_valid_std_header(&xsdt)) - return BAN::Error::from_error_code(ErrorCode::ACPI_RootInvalid); - - m_header_table_paddr = rsdp->xsdt_address + offsetof(XSDT, entries); - m_entry_size = 8; - root_entry_count = (xsdt.length - sizeof(SDTHeader)) / 8; + m_header_table_paddr = rsdp->xsdt_address + offsetof(XSDT, entries); + m_entry_size = 8; + root_entry_count = (xsdt.length - sizeof(SDTHeader)) / 8; + return {}; + } + )); } else { - PageTable::map_fast_page(rsdp->rsdt_address & PAGE_ADDR_MASK); - auto& rsdt = PageTable::fast_page_as(rsdp->rsdt_address % PAGE_SIZE); - BAN::ScopeGuard _([] { PageTable::unmap_fast_page(); }); + TRY(PageTable::with_fast_page(rsdp->rsdt_address & PAGE_ADDR_MASK, + [&]() -> BAN::ErrorOr + { + auto& rsdt = PageTable::fast_page_as(rsdp->rsdt_address % PAGE_SIZE); + if (memcmp(rsdt.signature, "RSDT", 4) != 0) + return BAN::Error::from_error_code(ErrorCode::ACPI_RootInvalid); + if (!is_valid_std_header(&rsdt)) + return BAN::Error::from_error_code(ErrorCode::ACPI_RootInvalid); - if (memcmp(rsdt.signature, "RSDT", 4) != 0) - return BAN::Error::from_error_code(ErrorCode::ACPI_RootInvalid); - if (!is_valid_std_header(&rsdt)) - return BAN::Error::from_error_code(ErrorCode::ACPI_RootInvalid); - - m_header_table_paddr = rsdp->rsdt_address + offsetof(RSDT, entries); - m_entry_size = 4; - root_entry_count = (rsdt.length - sizeof(SDTHeader)) / 4; + m_header_table_paddr = rsdp->rsdt_address + offsetof(RSDT, entries); + m_entry_size = 4; + root_entry_count = (rsdt.length - sizeof(SDTHeader)) / 4; + return {}; + } + )); } size_t needed_pages = range_page_count(m_header_table_paddr, root_entry_count * m_entry_size); @@ -144,9 +150,10 @@ namespace Kernel auto map_header = [](paddr_t header_paddr) -> vaddr_t { - PageTable::map_fast_page(header_paddr & PAGE_ADDR_MASK); - size_t header_length = PageTable::fast_page_as(header_paddr % PAGE_SIZE).length; - PageTable::unmap_fast_page(); + size_t header_length; + PageTable::with_fast_page(header_paddr & PAGE_ADDR_MASK, [&] { + header_length = PageTable::fast_page_as(header_paddr % PAGE_SIZE).length; + }); size_t needed_pages = range_page_count(header_paddr, header_length); vaddr_t page_vaddr = PageTable::kernel().reserve_free_contiguous_pages(needed_pages, KERNEL_OFFSET); diff --git a/kernel/kernel/APIC.cpp b/kernel/kernel/APIC.cpp index 6098f1e1..d08e41cc 100644 --- a/kernel/kernel/APIC.cpp +++ b/kernel/kernel/APIC.cpp @@ -1,8 +1,8 @@ #include -#include #include #include #include +#include #include #include #include diff --git a/kernel/kernel/Debug.cpp b/kernel/kernel/Debug.cpp index fd446579..c85d4e05 100644 --- a/kernel/kernel/Debug.cpp +++ b/kernel/kernel/Debug.cpp @@ -1,7 +1,7 @@ #include #include +#include #include -#include #include #include #include @@ -13,6 +13,8 @@ extern TerminalDriver* g_terminal_driver; namespace Debug { + Kernel::RecursiveSpinLock s_debug_lock; + void dump_stack_trace() { using namespace Kernel; @@ -120,18 +122,4 @@ namespace Debug BAN::Formatter::print(Debug::putchar, "[{5}.{3}] {}:{}: ", ms_since_boot / 1000, ms_since_boot % 1000, file, line); } - static Kernel::RecursiveSpinLock s_debug_lock; - - void DebugLock::lock() - { - if (Kernel::interrupts_enabled()) - s_debug_lock.lock(); - } - - void DebugLock::unlock() - { - if (Kernel::interrupts_enabled()) - s_debug_lock.unlock(); - } - } diff --git a/kernel/kernel/Device/DebugDevice.cpp b/kernel/kernel/Device/DebugDevice.cpp index 35615204..3d2af0ce 100644 --- a/kernel/kernel/Device/DebugDevice.cpp +++ b/kernel/kernel/Device/DebugDevice.cpp @@ -20,7 +20,7 @@ namespace Kernel BAN::ErrorOr DebugDevice::write_impl(off_t, BAN::ConstByteSpan buffer) { auto ms_since_boot = SystemTimer::get().ms_since_boot(); - Debug::DebugLock::lock(); + SpinLockGuard _(Debug::s_debug_lock); BAN::Formatter::print(Debug::putchar, "[{5}.{3}] {}: ", ms_since_boot / 1000, ms_since_boot % 1000, @@ -28,7 +28,6 @@ namespace Kernel ); for (size_t i = 0; i < buffer.size(); i++) Debug::putchar(buffer[i]); - Debug::DebugLock::unlock(); return buffer.size(); } diff --git a/kernel/kernel/FS/DevFS/FileSystem.cpp b/kernel/kernel/FS/DevFS/FileSystem.cpp index 2e615d20..426e7967 100644 --- a/kernel/kernel/FS/DevFS/FileSystem.cpp +++ b/kernel/kernel/FS/DevFS/FileSystem.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include #include @@ -56,12 +56,9 @@ namespace Kernel sync_process->add_thread(MUST(Thread::create_kernel( [](void*) { - // NOTE: we lock the device lock here and unlock - // it only while semaphore is blocking - s_instance->m_device_lock.lock(); - while (true) { + LockGuard _(s_instance->m_device_lock); while (!s_instance->m_should_sync) { LockFreeGuard _(s_instance->m_device_lock); diff --git a/kernel/kernel/FS/Ext2/FileSystem.cpp b/kernel/kernel/FS/Ext2/FileSystem.cpp index 88599076..0b3c91a0 100644 --- a/kernel/kernel/FS/Ext2/FileSystem.cpp +++ b/kernel/kernel/FS/Ext2/FileSystem.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #define EXT2_DEBUG_PRINT 0 #define EXT2_VERIFY_INODE 0 @@ -139,7 +139,7 @@ namespace Kernel BAN::ErrorOr Ext2FS::create_inode(const Ext2::Inode& ext2_inode) { - LockGuard _(m_lock); + LockGuard _(m_mutex); ASSERT(ext2_inode.size == 0); @@ -218,7 +218,7 @@ namespace Kernel void Ext2FS::delete_inode(uint32_t ino) { - LockGuard _(m_lock); + LockGuard _(m_mutex); ASSERT(ino >= superblock().first_ino); ASSERT(ino <= superblock().inodes_count); @@ -271,7 +271,7 @@ namespace Kernel void Ext2FS::read_block(uint32_t block, BlockBufferWrapper& buffer) { - LockGuard _(m_lock); + LockGuard _(m_mutex); const uint32_t sector_size = m_block_device->blksize(); const uint32_t block_size = this->block_size(); @@ -284,7 +284,7 @@ namespace Kernel void Ext2FS::write_block(uint32_t block, const BlockBufferWrapper& buffer) { - LockGuard _(m_lock); + LockGuard _(m_mutex); const uint32_t sector_size = m_block_device->blksize(); const uint32_t block_size = this->block_size(); @@ -297,7 +297,7 @@ namespace Kernel void Ext2FS::sync_superblock() { - LockGuard _(m_lock); + LockGuard _(m_mutex); const uint32_t sector_size = m_block_device->blksize(); ASSERT(1024 % sector_size == 0); @@ -322,13 +322,13 @@ namespace Kernel Ext2FS::BlockBufferWrapper Ext2FS::get_block_buffer() { - LockGuard _(m_lock); + LockGuard _(m_mutex); return m_buffer_manager.get_buffer(); } BAN::ErrorOr Ext2FS::reserve_free_block(uint32_t primary_bgd) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (m_superblock.r_blocks_count >= m_superblock.free_blocks_count) return BAN::Error::from_errno(ENOSPC); @@ -389,7 +389,7 @@ namespace Kernel void Ext2FS::release_block(uint32_t block) { - LockGuard _(m_lock); + LockGuard _(m_mutex); ASSERT(block >= m_superblock.first_data_block); ASSERT(block < m_superblock.blocks_count); @@ -422,7 +422,7 @@ namespace Kernel Ext2FS::BlockLocation Ext2FS::locate_inode(uint32_t ino) { - LockGuard _(m_lock); + LockGuard _(m_mutex); ASSERT(ino <= superblock().inodes_count); @@ -464,7 +464,7 @@ namespace Kernel Ext2FS::BlockLocation Ext2FS::locate_block_group_descriptior(uint32_t group_index) { - LockGuard _(m_lock); + LockGuard _(m_mutex); const uint32_t block_size = this->block_size(); diff --git a/kernel/kernel/FS/Inode.cpp b/kernel/kernel/FS/Inode.cpp index 74702e49..b7341f0f 100644 --- a/kernel/kernel/FS/Inode.cpp +++ b/kernel/kernel/FS/Inode.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include @@ -58,13 +58,13 @@ namespace Kernel void Inode::on_close() { - LockGuard _(m_lock); + LockGuard _(m_mutex); on_close_impl(); } BAN::ErrorOr> Inode::find_inode(BAN::StringView name) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (!mode().ifdir()) return BAN::Error::from_errno(ENOTDIR); return find_inode_impl(name); @@ -72,7 +72,7 @@ namespace Kernel BAN::ErrorOr Inode::list_next_inodes(off_t offset, DirectoryEntryList* list, size_t list_len) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (!mode().ifdir()) return BAN::Error::from_errno(ENOTDIR); return list_next_inodes_impl(offset, list, list_len); @@ -80,7 +80,7 @@ namespace Kernel BAN::ErrorOr Inode::create_file(BAN::StringView name, mode_t mode, uid_t uid, gid_t gid) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (!this->mode().ifdir()) return BAN::Error::from_errno(ENOTDIR); if (Mode(mode).ifdir()) @@ -90,7 +90,7 @@ namespace Kernel BAN::ErrorOr Inode::create_directory(BAN::StringView name, mode_t mode, uid_t uid, gid_t gid) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (!this->mode().ifdir()) return BAN::Error::from_errno(ENOTDIR); if (!Mode(mode).ifdir()) @@ -100,7 +100,7 @@ namespace Kernel BAN::ErrorOr Inode::unlink(BAN::StringView name) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (!mode().ifdir()) return BAN::Error::from_errno(ENOTDIR); if (name == "."sv || name == ".."sv) @@ -110,7 +110,7 @@ namespace Kernel BAN::ErrorOr Inode::link_target() { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (!mode().iflnk()) return BAN::Error::from_errno(EINVAL); return link_target_impl(); @@ -118,7 +118,7 @@ namespace Kernel BAN::ErrorOr Inode::accept(sockaddr* address, socklen_t* address_len) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (!mode().ifsock()) return BAN::Error::from_errno(ENOTSOCK); return accept_impl(address, address_len); @@ -126,7 +126,7 @@ namespace Kernel BAN::ErrorOr Inode::bind(const sockaddr* address, socklen_t address_len) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (!mode().ifsock()) return BAN::Error::from_errno(ENOTSOCK); return bind_impl(address, address_len); @@ -134,7 +134,7 @@ namespace Kernel BAN::ErrorOr Inode::connect(const sockaddr* address, socklen_t address_len) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (!mode().ifsock()) return BAN::Error::from_errno(ENOTSOCK); return connect_impl(address, address_len); @@ -142,7 +142,7 @@ namespace Kernel BAN::ErrorOr Inode::listen(int backlog) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (!mode().ifsock()) return BAN::Error::from_errno(ENOTSOCK); return listen_impl(backlog); @@ -150,7 +150,7 @@ namespace Kernel BAN::ErrorOr Inode::sendto(BAN::ConstByteSpan message, const sockaddr* address, socklen_t address_len) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (!mode().ifsock()) return BAN::Error::from_errno(ENOTSOCK); return sendto_impl(message, address, address_len); @@ -158,7 +158,7 @@ namespace Kernel BAN::ErrorOr Inode::recvfrom(BAN::ByteSpan buffer, sockaddr* address, socklen_t* address_len) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (!mode().ifsock()) return BAN::Error::from_errno(ENOTSOCK); return recvfrom_impl(buffer, address, address_len); @@ -166,7 +166,7 @@ namespace Kernel BAN::ErrorOr Inode::read(off_t offset, BAN::ByteSpan buffer) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (mode().ifdir()) return BAN::Error::from_errno(EISDIR); return read_impl(offset, buffer); @@ -174,7 +174,7 @@ namespace Kernel BAN::ErrorOr Inode::write(off_t offset, BAN::ConstByteSpan buffer) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (mode().ifdir()) return BAN::Error::from_errno(EISDIR); return write_impl(offset, buffer); @@ -182,7 +182,7 @@ namespace Kernel BAN::ErrorOr Inode::truncate(size_t size) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (mode().ifdir()) return BAN::Error::from_errno(EISDIR); return truncate_impl(size); @@ -191,37 +191,37 @@ namespace Kernel BAN::ErrorOr Inode::chmod(mode_t mode) { ASSERT((mode & Inode::Mode::TYPE_MASK) == 0); - LockGuard _(m_lock); + LockGuard _(m_mutex); return chmod_impl(mode); } BAN::ErrorOr Inode::chown(uid_t uid, gid_t gid) { - LockGuard _(m_lock); + LockGuard _(m_mutex); return chown_impl(uid, gid); } bool Inode::can_read() const { - LockGuard _(m_lock); + LockGuard _(m_mutex); return can_read_impl(); } bool Inode::can_write() const { - LockGuard _(m_lock); + LockGuard _(m_mutex); return can_write_impl(); } bool Inode::has_error() const { - LockGuard _(m_lock); + LockGuard _(m_mutex); return has_error_impl(); } BAN::ErrorOr Inode::ioctl(int request, void* arg) { - LockGuard _(m_lock); + LockGuard _(m_mutex); return ioctl_impl(request, arg); } diff --git a/kernel/kernel/FS/Pipe.cpp b/kernel/kernel/FS/Pipe.cpp index af70a75f..8d159b5c 100644 --- a/kernel/kernel/FS/Pipe.cpp +++ b/kernel/kernel/FS/Pipe.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include @@ -26,14 +26,14 @@ namespace Kernel void Pipe::clone_writing() { - LockGuard _(m_lock); + LockGuard _(m_mutex); ASSERT(m_writing_count > 0); m_writing_count++; } void Pipe::close_writing() { - LockGuard _(m_lock); + LockGuard _(m_mutex); ASSERT(m_writing_count > 0); m_writing_count--; if (m_writing_count == 0) @@ -42,12 +42,12 @@ namespace Kernel BAN::ErrorOr Pipe::read_impl(off_t, BAN::ByteSpan buffer) { - LockGuard _(m_lock); + LockGuard _(m_mutex); while (m_buffer.empty()) { if (m_writing_count == 0) return 0; - LockFreeGuard lock_free(m_lock); + LockFreeGuard lock_free(m_mutex); TRY(Thread::current().block_or_eintr_indefinite(m_semaphore)); } @@ -66,7 +66,7 @@ namespace Kernel BAN::ErrorOr Pipe::write_impl(off_t, BAN::ConstByteSpan buffer) { - LockGuard _(m_lock); + LockGuard _(m_mutex); size_t old_size = m_buffer.size(); diff --git a/kernel/kernel/FS/ProcFS/FileSystem.cpp b/kernel/kernel/FS/ProcFS/FileSystem.cpp index d5400582..1fa82a9e 100644 --- a/kernel/kernel/FS/ProcFS/FileSystem.cpp +++ b/kernel/kernel/FS/ProcFS/FileSystem.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include namespace Kernel { diff --git a/kernel/kernel/FS/TmpFS/FileSystem.cpp b/kernel/kernel/FS/TmpFS/FileSystem.cpp index 582afb35..bc94ac50 100644 --- a/kernel/kernel/FS/TmpFS/FileSystem.cpp +++ b/kernel/kernel/FS/TmpFS/FileSystem.cpp @@ -66,7 +66,7 @@ namespace Kernel BAN::ErrorOr> TmpFileSystem::open_inode(ino_t ino) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (m_inode_cache.contains(ino)) return m_inode_cache[ino]; @@ -85,7 +85,7 @@ namespace Kernel BAN::ErrorOr TmpFileSystem::add_to_cache(BAN::RefPtr inode) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (!m_inode_cache.contains(inode->ino())) TRY(m_inode_cache.insert(inode->ino(), inode)); @@ -94,7 +94,7 @@ namespace Kernel void TmpFileSystem::remove_from_cache(BAN::RefPtr inode) { - LockGuard _(m_lock); + LockGuard _(m_mutex); ASSERT(m_inode_cache.contains(inode->ino())); m_inode_cache.remove(inode->ino()); @@ -102,7 +102,7 @@ namespace Kernel void TmpFileSystem::read_inode(ino_t ino, TmpInodeInfo& out) { - LockGuard _(m_lock); + LockGuard _(m_mutex); auto inode_location = find_inode(ino); PageTable::with_fast_page(inode_location.paddr, [&] { @@ -112,7 +112,7 @@ namespace Kernel void TmpFileSystem::write_inode(ino_t ino, const TmpInodeInfo& info) { - LockGuard _(m_lock); + LockGuard _(m_mutex); auto inode_location = find_inode(ino); PageTable::with_fast_page(inode_location.paddr, [&] { @@ -123,7 +123,7 @@ namespace Kernel void TmpFileSystem::delete_inode(ino_t ino) { - LockGuard _(m_lock); + LockGuard _(m_mutex); auto inode_location = find_inode(ino); PageTable::with_fast_page(inode_location.paddr, [&] { @@ -138,7 +138,7 @@ namespace Kernel BAN::ErrorOr TmpFileSystem::allocate_inode(const TmpInodeInfo& info) { - LockGuard _(m_lock); + LockGuard _(m_mutex); constexpr size_t inodes_per_page = PAGE_SIZE / sizeof(TmpInodeInfo); @@ -164,7 +164,7 @@ namespace Kernel TmpFileSystem::InodeLocation TmpFileSystem::find_inode(ino_t ino) { - LockGuard _(m_lock); + LockGuard _(m_mutex); ASSERT_GTE(ino, first_inode); ASSERT_LT(ino, max_inodes); @@ -182,7 +182,7 @@ namespace Kernel void TmpFileSystem::free_block(size_t index) { - LockGuard _(m_lock); + LockGuard _(m_mutex); constexpr size_t addresses_per_page = PAGE_SIZE / sizeof(PageInfo); @@ -204,7 +204,7 @@ namespace Kernel BAN::ErrorOr TmpFileSystem::allocate_block() { - LockGuard _(m_lock); + LockGuard _(m_mutex); size_t result = first_data_page; TRY(for_each_indirect_paddr_allocating(m_data_pages, [&] (paddr_t, bool allocated) { @@ -218,7 +218,7 @@ namespace Kernel paddr_t TmpFileSystem::find_block(size_t index) { - LockGuard _(m_lock); + LockGuard _(m_mutex); ASSERT_GT(index, 0); return find_indirect(m_data_pages, index - first_data_page, 3); @@ -226,7 +226,7 @@ namespace Kernel paddr_t TmpFileSystem::find_indirect(PageInfo root, size_t index, size_t depth) { - LockGuard _(m_lock); + LockGuard _(m_mutex); ASSERT(root.flags() & PageInfo::Flags::Present); if (depth == 0) @@ -257,7 +257,7 @@ namespace Kernel template BAN::ErrorOr TmpFileSystem::for_each_indirect_paddr_allocating_internal(PageInfo page_info, F callback, size_t depth) { - LockGuard _(m_lock); + LockGuard _(m_mutex); ASSERT(page_info.flags() & PageInfo::Flags::Present); if (depth == 0) @@ -316,7 +316,7 @@ namespace Kernel template BAN::ErrorOr TmpFileSystem::for_each_indirect_paddr_allocating(PageInfo page_info, F callback, size_t depth) { - LockGuard _(m_lock); + LockGuard _(m_mutex); BAN::Iteration result = TRY(for_each_indirect_paddr_allocating_internal(page_info, callback, depth)); ASSERT(result == BAN::Iteration::Break); diff --git a/kernel/kernel/FS/VirtualFileSystem.cpp b/kernel/kernel/FS/VirtualFileSystem.cpp index 36ba3426..cf1a66ea 100644 --- a/kernel/kernel/FS/VirtualFileSystem.cpp +++ b/kernel/kernel/FS/VirtualFileSystem.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include namespace Kernel @@ -63,15 +63,14 @@ namespace Kernel if (!file.inode->mode().ifdir()) return BAN::Error::from_errno(ENOTDIR); - LockGuard _(m_lock); + LockGuard _(m_mutex); TRY(m_mount_points.push_back({ file, file_system })); - return {}; } VirtualFileSystem::MountPoint* VirtualFileSystem::mount_from_host_inode(BAN::RefPtr inode) { - ASSERT(m_lock.is_locked()); + LockGuard _(m_mutex); for (MountPoint& mount : m_mount_points) if (*mount.host.inode == *inode) return &mount; @@ -80,7 +79,7 @@ namespace Kernel VirtualFileSystem::MountPoint* VirtualFileSystem::mount_from_root_inode(BAN::RefPtr inode) { - ASSERT(m_lock.is_locked()); + LockGuard _(m_mutex); for (MountPoint& mount : m_mount_points) if (*mount.target->root_inode() == *inode) return &mount; @@ -89,7 +88,7 @@ namespace Kernel BAN::ErrorOr VirtualFileSystem::file_from_absolute_path(const Credentials& credentials, BAN::StringView path, int flags) { - LockGuard _(m_lock); + LockGuard _(m_mutex); ASSERT(path.front() == '/'); diff --git a/kernel/kernel/Input/PS2/Controller.cpp b/kernel/kernel/Input/PS2/Controller.cpp index 9f6fc1d5..875a9d79 100644 --- a/kernel/kernel/Input/PS2/Controller.cpp +++ b/kernel/kernel/Input/PS2/Controller.cpp @@ -22,7 +22,7 @@ namespace Kernel::Input BAN::ErrorOr PS2Controller::send_byte(uint16_t port, uint8_t byte) { ASSERT(interrupts_enabled()); - LockGuard _(m_lock); + LockGuard _(m_mutex); uint64_t timeout = SystemTimer::get().ms_since_boot() + s_ps2_timeout_ms; while (SystemTimer::get().ms_since_boot() < timeout) { @@ -37,7 +37,7 @@ namespace Kernel::Input BAN::ErrorOr PS2Controller::read_byte() { ASSERT(interrupts_enabled()); - LockGuard _(m_lock); + LockGuard _(m_mutex); uint64_t timeout = SystemTimer::get().ms_since_boot() + s_ps2_timeout_ms; while (SystemTimer::get().ms_since_boot() < timeout) { @@ -50,14 +50,14 @@ namespace Kernel::Input BAN::ErrorOr PS2Controller::send_command(PS2::Command command) { - LockGuard _(m_lock); + LockGuard _(m_mutex); TRY(send_byte(PS2::IOPort::COMMAND, command)); return {}; } BAN::ErrorOr PS2Controller::send_command(PS2::Command command, uint8_t data) { - LockGuard _(m_lock); + LockGuard _(m_mutex); TRY(send_byte(PS2::IOPort::COMMAND, command)); TRY(send_byte(PS2::IOPort::DATA, data)); return {}; @@ -65,7 +65,7 @@ namespace Kernel::Input BAN::ErrorOr PS2Controller::device_send_byte(uint8_t device_index, uint8_t byte) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (device_index == 1) TRY(send_byte(PS2::IOPort::COMMAND, PS2::Command::WRITE_TO_SECOND_PORT)); TRY(send_byte(PS2::IOPort::DATA, byte)); @@ -74,7 +74,7 @@ namespace Kernel::Input BAN::ErrorOr PS2Controller::device_send_byte_and_wait_ack(uint8_t device_index, uint8_t byte) { - LockGuard _(m_lock); + LockGuard _(m_mutex); for (;;) { TRY(device_send_byte(device_index, byte)); diff --git a/kernel/kernel/InterruptController.cpp b/kernel/kernel/InterruptController.cpp index 4b21bc95..c8e1f312 100644 --- a/kernel/kernel/InterruptController.cpp +++ b/kernel/kernel/InterruptController.cpp @@ -67,11 +67,4 @@ namespace Kernel dwarnln("could not enter acpi mode"); } - bool interrupts_enabled() - { - uintptr_t flags; - asm volatile("pushf; pop %0" : "=r"(flags) :: "memory"); - return flags & (1 << 9); - } - } diff --git a/kernel/kernel/Lock/SpinLock.cpp b/kernel/kernel/Lock/SpinLock.cpp new file mode 100644 index 00000000..771e330d --- /dev/null +++ b/kernel/kernel/Lock/SpinLock.cpp @@ -0,0 +1,64 @@ +#include +#include +#include + +// FIXME: try to move these to header + +namespace Kernel +{ + + InterruptState SpinLock::lock() + { + auto tid = Scheduler::current_tid(); + ASSERT_NEQ(m_locker.load(), tid); + + InterruptState state = interrupts_enabled(); + DISABLE_INTERRUPTS(); + + if (!m_locker.compare_exchange(-1, tid)) + ASSERT_NOT_REACHED(); + + return state; + } + + void SpinLock::unlock(InterruptState state) + { + ASSERT_EQ(m_locker.load(), Scheduler::current_tid()); + m_locker.store(-1); + if (state) + ENABLE_INTERRUPTS(); + } + + InterruptState RecursiveSpinLock::lock() + { + auto tid = Scheduler::current_tid(); + + InterruptState state = interrupts_enabled(); + DISABLE_INTERRUPTS(); + + if (tid == m_locker) + ASSERT_GT(m_lock_depth, 0); + else + { + if (!m_locker.compare_exchange(-1, tid)) + ASSERT_NOT_REACHED(); + ASSERT_EQ(m_lock_depth, 0); + } + + m_lock_depth++; + + return state; + } + + void RecursiveSpinLock::unlock(InterruptState state) + { + auto tid = Scheduler::current_tid(); + ASSERT_EQ(m_locker.load(), tid); + ASSERT_GT(m_lock_depth, 0); + if (--m_lock_depth == 0) + m_locker = -1; + if (state) + ENABLE_INTERRUPTS(); + } + +} diff --git a/kernel/kernel/Memory/FileBackedRegion.cpp b/kernel/kernel/Memory/FileBackedRegion.cpp index 22e5d649..8156265d 100644 --- a/kernel/kernel/Memory/FileBackedRegion.cpp +++ b/kernel/kernel/Memory/FileBackedRegion.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include @@ -26,7 +26,7 @@ namespace Kernel if (type == Type::SHARED) { - LockGuard _(inode->m_lock); + LockGuard _(inode->m_mutex); if (inode->m_shared_region.valid()) region->m_shared_data = inode->m_shared_region.lock(); else @@ -157,7 +157,7 @@ namespace Kernel } else if (m_type == Type::SHARED) { - LockGuard _(m_inode->m_lock); + LockGuard _(m_inode->m_mutex); ASSERT(m_inode->m_shared_region.valid()); ASSERT(m_shared_data->pages.size() == BAN::Math::div_round_up(m_inode->size(), PAGE_SIZE)); @@ -175,10 +175,9 @@ namespace Kernel TRY(m_inode->read(offset, BAN::ByteSpan(m_shared_data->page_buffer, bytes))); - CriticalScope _; - PageTable::map_fast_page(pages[page_index]); - memcpy(PageTable::fast_page_as_ptr(), m_shared_data->page_buffer, PAGE_SIZE); - PageTable::unmap_fast_page(); + PageTable::with_fast_page(pages[page_index], [&] { + memcpy(PageTable::fast_page_as_ptr(), m_shared_data->page_buffer, PAGE_SIZE); + }); } paddr_t paddr = pages[page_index]; diff --git a/kernel/kernel/Memory/Heap.cpp b/kernel/kernel/Memory/Heap.cpp index bc5d51d6..0db22c96 100644 --- a/kernel/kernel/Memory/Heap.cpp +++ b/kernel/kernel/Memory/Heap.cpp @@ -1,5 +1,4 @@ #include -#include #include #include @@ -67,7 +66,7 @@ namespace Kernel paddr_t Heap::take_free_page() { - LockGuard _(m_lock); + SpinLockGuard _(m_lock); for (auto& range : m_physical_ranges) if (range.free_pages() >= 1) return range.reserve_page(); @@ -76,7 +75,7 @@ namespace Kernel void Heap::release_page(paddr_t paddr) { - LockGuard _(m_lock); + SpinLockGuard _(m_lock); for (auto& range : m_physical_ranges) if (range.contains(paddr)) return range.release_page(paddr); @@ -85,7 +84,7 @@ namespace Kernel paddr_t Heap::take_free_contiguous_pages(size_t pages) { - LockGuard _(m_lock); + SpinLockGuard _(m_lock); for (auto& range : m_physical_ranges) if (range.free_pages() >= pages) if (paddr_t paddr = range.reserve_contiguous_pages(pages)) @@ -95,7 +94,7 @@ namespace Kernel void Heap::release_contiguous_pages(paddr_t paddr, size_t pages) { - LockGuard _(m_lock); + SpinLockGuard _(m_lock); for (auto& range : m_physical_ranges) if (range.contains(paddr)) return range.release_contiguous_pages(paddr, pages); @@ -104,7 +103,7 @@ namespace Kernel size_t Heap::used_pages() const { - LockGuard _(m_lock); + SpinLockGuard _(m_lock); size_t result = 0; for (const auto& range : m_physical_ranges) result += range.used_pages(); @@ -113,7 +112,7 @@ namespace Kernel size_t Heap::free_pages() const { - LockGuard _(m_lock); + SpinLockGuard _(m_lock); size_t result = 0; for (const auto& range : m_physical_ranges) result += range.free_pages(); diff --git a/kernel/kernel/Memory/MemoryBackedRegion.cpp b/kernel/kernel/Memory/MemoryBackedRegion.cpp index 3b7066eb..549b79fb 100644 --- a/kernel/kernel/Memory/MemoryBackedRegion.cpp +++ b/kernel/kernel/Memory/MemoryBackedRegion.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include @@ -57,15 +57,9 @@ namespace Kernel m_page_table.map_page_at(paddr, vaddr, m_flags); // Zero out the new page - if (&PageTable::current() == &m_page_table) - memset((void*)vaddr, 0x00, PAGE_SIZE); - else - { - CriticalScope _; - PageTable::map_fast_page(paddr); + PageTable::with_fast_page(paddr, [&] { memset(PageTable::fast_page_as_ptr(), 0x00, PAGE_SIZE); - PageTable::unmap_fast_page(); - } + }); return true; } @@ -100,15 +94,9 @@ namespace Kernel TRY(allocate_page_containing(write_vaddr)); - if (&PageTable::current() == &m_page_table) - memcpy((void*)write_vaddr, (void*)(buffer + written), bytes); - else - { - CriticalScope _; - PageTable::map_fast_page(m_page_table.physical_address_of(write_vaddr & PAGE_ADDR_MASK)); + PageTable::with_fast_page(m_page_table.physical_address_of(write_vaddr & PAGE_ADDR_MASK), [&] { memcpy(PageTable::fast_page_as_ptr(page_offset), (void*)(buffer + written), bytes); - PageTable::unmap_fast_page(); - } + }); written += bytes; } diff --git a/kernel/kernel/Memory/VirtualRange.cpp b/kernel/kernel/Memory/VirtualRange.cpp index 61f3b812..b70bb933 100644 --- a/kernel/kernel/Memory/VirtualRange.cpp +++ b/kernel/kernel/Memory/VirtualRange.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include @@ -68,7 +68,7 @@ namespace Kernel } ASSERT(vaddr + size <= vaddr_end); - LockGuard _(page_table); + SpinLockGuard _(page_table); page_table.unmap_range(vaddr, size); // We have to unmap here to allow reservation in create_to_vaddr() return create_to_vaddr(page_table, vaddr, size, flags, preallocate_pages); } @@ -99,7 +99,7 @@ namespace Kernel auto result = TRY(create_to_vaddr(page_table, vaddr(), size(), flags(), m_preallocated)); - LockGuard _(m_page_table); + SpinLockGuard _(m_page_table); for (size_t offset = 0; offset < size(); offset += PAGE_SIZE) { if (!m_preallocated && m_page_table.physical_address_of(vaddr() + offset)) @@ -110,10 +110,9 @@ namespace Kernel result->m_page_table.map_page_at(paddr, vaddr() + offset, m_flags); } - CriticalScope _; - PageTable::map_fast_page(result->m_page_table.physical_address_of(vaddr() + offset)); - memcpy(PageTable::fast_page_as_ptr(), (void*)(vaddr() + offset), PAGE_SIZE); - PageTable::unmap_fast_page(); + PageTable::with_fast_page(result->m_page_table.physical_address_of(vaddr() + offset), [&] { + memcpy(PageTable::fast_page_as_ptr(), (void*)(vaddr() + offset), PAGE_SIZE); + }); } return result; @@ -148,10 +147,9 @@ namespace Kernel for (size_t offset = 0; offset < size(); offset += PAGE_SIZE) { - CriticalScope _; - PageTable::map_fast_page(m_page_table.physical_address_of(vaddr() + offset)); - memset(PageTable::fast_page_as_ptr(), 0x00, PAGE_SIZE); - PageTable::unmap_fast_page(); + PageTable::with_fast_page(m_page_table.physical_address_of(vaddr() + offset), [&] { + memset(PageTable::fast_page_as_ptr(), 0x00, PAGE_SIZE); + }); } } @@ -176,12 +174,9 @@ namespace Kernel while (bytes > 0) { - { - CriticalScope _; - PageTable::map_fast_page(m_page_table.physical_address_of(vaddr() + page_index * PAGE_SIZE)); + PageTable::with_fast_page(m_page_table.physical_address_of(vaddr() + page_index * PAGE_SIZE), [&] { memcpy(PageTable::fast_page_as_ptr(page_offset), buffer, PAGE_SIZE - page_offset); - PageTable::unmap_fast_page(); - } + }); buffer += PAGE_SIZE - page_offset; bytes -= PAGE_SIZE - page_offset; diff --git a/kernel/kernel/Networking/ARPTable.cpp b/kernel/kernel/Networking/ARPTable.cpp index 51e8387c..19aacaf5 100644 --- a/kernel/kernel/Networking/ARPTable.cpp +++ b/kernel/kernel/Networking/ARPTable.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -52,9 +51,10 @@ namespace Kernel ipv4_address = interface.get_gateway(); { - LockGuard _(m_lock); - if (m_arp_table.contains(ipv4_address)) - return m_arp_table[ipv4_address]; + SpinLockGuard _(m_table_lock); + auto it = m_arp_table.find(ipv4_address); + if (it != m_arp_table.end()) + return it->value; } ARPPacket arp_request; @@ -74,9 +74,10 @@ namespace Kernel while (SystemTimer::get().ms_since_boot() < timeout) { { - LockGuard _(m_lock); - if (m_arp_table.contains(ipv4_address)) - return m_arp_table[ipv4_address]; + SpinLockGuard _(m_table_lock); + auto it = m_arp_table.find(ipv4_address); + if (it != m_arp_table.end()) + return it->value; } Scheduler::get().reschedule(); } @@ -114,13 +115,15 @@ namespace Kernel } case ARPOperation::Reply: { - LockGuard _(m_lock); - if (m_arp_table.contains(packet.spa)) + SpinLockGuard _(m_table_lock); + auto it = m_arp_table.find(packet.spa); + + if (it != m_arp_table.end()) { - if (m_arp_table[packet.spa] != packet.sha) + if (it->value != packet.sha) { dprintln("Update IPv4 {} MAC to {}", packet.spa, packet.sha); - m_arp_table[packet.spa] = packet.sha; + it->value = packet.sha; } } else diff --git a/kernel/kernel/Networking/IPv4Layer.cpp b/kernel/kernel/Networking/IPv4Layer.cpp index 00f2f217..fbe3ca88 100644 --- a/kernel/kernel/Networking/IPv4Layer.cpp +++ b/kernel/kernel/Networking/IPv4Layer.cpp @@ -70,12 +70,14 @@ namespace Kernel void IPv4Layer::unbind_socket(BAN::RefPtr socket, uint16_t port) { - LockGuard _(m_lock); - if (m_bound_sockets.contains(port)) { - ASSERT(m_bound_sockets[port].valid()); - ASSERT(m_bound_sockets[port].lock() == socket); - m_bound_sockets.remove(port); + SpinLockGuard _(m_bound_socket_lock); + auto it = m_bound_sockets.find(port); + if (it != m_bound_sockets.end()) + { + ASSERT(it->value.lock() == socket); + m_bound_sockets.remove(it); + } } NetworkManager::get().TmpFileSystem::remove_from_cache(socket); } @@ -88,7 +90,7 @@ namespace Kernel return BAN::Error::from_errno(EAFNOSUPPORT); auto& sockaddr_in = *reinterpret_cast(address); - LockGuard _(m_lock); + SpinLockGuard _(m_bound_socket_lock); uint16_t port = NetworkSocket::PORT_NONE; for (uint32_t i = 0; i < 100 && port == NetworkSocket::PORT_NONE; i++) @@ -124,7 +126,7 @@ namespace Kernel auto& sockaddr_in = *reinterpret_cast(address); uint16_t port = BAN::host_to_network_endian(sockaddr_in.sin_port); - LockGuard _(m_lock); + SpinLockGuard _(m_bound_socket_lock); if (m_bound_sockets.contains(port)) return BAN::Error::from_errno(EADDRINUSE); @@ -250,13 +252,14 @@ namespace Kernel BAN::RefPtr bound_socket; { - LockGuard _(m_lock); - if (!m_bound_sockets.contains(dst_port)) + SpinLockGuard _(m_bound_socket_lock); + auto it = m_bound_sockets.find(dst_port); + if (it == m_bound_sockets.end()) { dprintln_if(DEBUG_IPV4, "no one is listening on port {}", dst_port); return {}; } - bound_socket = m_bound_sockets[dst_port].lock(); + bound_socket = it->value.lock(); } if (!bound_socket) diff --git a/kernel/kernel/Networking/TCPSocket.cpp b/kernel/kernel/Networking/TCPSocket.cpp index 7cda7baa..18c46708 100644 --- a/kernel/kernel/Networking/TCPSocket.cpp +++ b/kernel/kernel/Networking/TCPSocket.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -68,7 +68,7 @@ namespace Kernel void TCPSocket::on_close_impl() { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (!is_bound()) return; @@ -103,7 +103,7 @@ namespace Kernel if (address_len > (socklen_t)sizeof(sockaddr_storage)) address_len = sizeof(sockaddr_storage); - LockGuard _(m_lock); + LockGuard _(m_mutex); ASSERT(!m_connection_info.has_value()); @@ -139,7 +139,7 @@ namespace Kernel uint64_t wake_time_ms = SystemTimer::get().ms_since_boot() + 5000; while (m_state != State::Established) { - LockFreeGuard free(m_lock); + LockFreeGuard free(m_mutex); if (SystemTimer::get().ms_since_boot() >= wake_time_ms) return BAN::Error::from_errno(ECONNREFUSED); TRY(Thread::current().block_or_eintr_or_waketime(m_semaphore, wake_time_ms, true)); @@ -212,7 +212,7 @@ namespace Kernel { case State::Closed: { - LockGuard _(m_lock); + LockGuard _(m_mutex); header.syn = 1; add_tcp_header_option<0, TCPOption::MaximumSeqmentSize>(header, m_interface->payload_mtu() - m_network_layer.header_size()); add_tcp_header_option<4, TCPOption::WindowScale>(header, 0); @@ -233,7 +233,7 @@ namespace Kernel break; case State::CloseWait: { - LockGuard _(m_lock); + LockGuard _(m_mutex); header.ack = 1; header.fin = 1; m_state = State::LastAck; @@ -242,7 +242,7 @@ namespace Kernel } case State::FinWait1: { - LockGuard _(m_lock); + LockGuard _(m_mutex); header.ack = 1; header.fin = 1; m_state = State::FinWait2; @@ -250,7 +250,7 @@ namespace Kernel } case State::FinWait2: { - LockGuard _(m_lock); + LockGuard _(m_mutex); header.ack = 1; m_state = State::TimeWait; m_time_wait_start_ms = SystemTimer::get().ms_since_boot(); @@ -312,7 +312,7 @@ namespace Kernel if (!header.ack || !header.syn) break; - LockGuard _(m_lock); + LockGuard _(m_mutex); if (header.ack_number != m_send_window.current_seq) { @@ -345,7 +345,7 @@ namespace Kernel if (!header.ack) break; - LockGuard _(m_lock); + LockGuard _(m_mutex); if (header.fin) { @@ -436,7 +436,7 @@ namespace Kernel set_connection_as_closed(); { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (m_should_ack) { @@ -518,7 +518,7 @@ namespace Kernel BAN::ErrorOr TCPSocket::recvfrom_impl(BAN::ByteSpan buffer, sockaddr*, socklen_t*) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (m_state == State::Closed) return BAN::Error::from_errno(ENOTCONN); @@ -542,7 +542,7 @@ namespace Kernel case State::Closing: ASSERT_NOT_REACHED(); }; - LockFreeGuard free(m_lock); + LockFreeGuard free(m_mutex); TRY(Thread::current().block_or_eintr_indefinite(m_semaphore)); } @@ -575,7 +575,7 @@ namespace Kernel return message.size(); } - LockGuard _(m_lock); + LockGuard _(m_mutex); if (m_state == State::Closed) return BAN::Error::from_errno(ENOTCONN); @@ -602,7 +602,7 @@ namespace Kernel if (m_send_window.data_size + message.size() <= m_send_window.buffer->size()) break; - LockFreeGuard free(m_lock); + LockFreeGuard free(m_mutex); TRY(Thread::current().block_or_eintr_indefinite(m_semaphore)); } @@ -634,7 +634,7 @@ namespace Kernel case State::Closing: ASSERT_NOT_REACHED(); }; - LockFreeGuard free(m_lock); + LockFreeGuard free(m_mutex); TRY(Thread::current().block_or_eintr_indefinite(m_semaphore)); } diff --git a/kernel/kernel/Networking/UDPSocket.cpp b/kernel/kernel/Networking/UDPSocket.cpp index f49e5bd9..6345b694 100644 --- a/kernel/kernel/Networking/UDPSocket.cpp +++ b/kernel/kernel/Networking/UDPSocket.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -46,7 +45,7 @@ namespace Kernel //auto& header = packet.as(); auto payload = packet.slice(sizeof(UDPHeader)); - LockGuard _(m_packet_lock); + SpinLockGuard _(m_packet_lock); if (m_packets.full()) { @@ -88,12 +87,12 @@ namespace Kernel } ASSERT(m_port != PORT_NONE); - LockGuard _(m_packet_lock); - + auto state = m_packet_lock.lock(); while (m_packets.empty()) { - LockFreeGuard free(m_packet_lock); + m_packet_lock.unlock(state); TRY(Thread::current().block_or_eintr_indefinite(m_packet_semaphore)); + state = m_packet_lock.lock(); } auto packet_info = m_packets.front(); @@ -115,6 +114,8 @@ namespace Kernel m_packet_total_size -= packet_info.packet_size; + m_packet_lock.unlock(state); + if (address && address_len) { if (*address_len > (socklen_t)sizeof(sockaddr_storage)) diff --git a/kernel/kernel/Networking/UNIX/Socket.cpp b/kernel/kernel/Networking/UNIX/Socket.cpp index 3f8eb1f4..e986cbc1 100644 --- a/kernel/kernel/Networking/UNIX/Socket.cpp +++ b/kernel/kernel/Networking/UNIX/Socket.cpp @@ -51,9 +51,10 @@ namespace Kernel { if (is_bound() && !is_bound_to_unused()) { - LockGuard _(s_bound_socket_lock); - if (s_bound_sockets.contains(m_bound_path)) - s_bound_sockets.remove(m_bound_path); + SpinLockGuard _(s_bound_socket_lock); + auto it = s_bound_sockets.find(m_bound_path); + if (it != s_bound_sockets.end()) + s_bound_sockets.remove(it); } } @@ -71,7 +72,7 @@ namespace Kernel BAN::RefPtr pending; { - LockGuard _(connection_info.pending_lock); + SpinLockGuard _(connection_info.pending_lock); pending = connection_info.pending_connections.front(); connection_info.pending_connections.pop(); connection_info.pending_semaphore.unblock(); @@ -120,10 +121,11 @@ namespace Kernel BAN::RefPtr target; { - LockGuard _(s_bound_socket_lock); - if (!s_bound_sockets.contains(file.canonical_path)) + SpinLockGuard _(s_bound_socket_lock); + auto it = s_bound_sockets.find(file.canonical_path); + if (it == s_bound_sockets.end()) return BAN::Error::from_errno(ECONNREFUSED); - target = s_bound_sockets[file.canonical_path].lock(); + target = it->value.lock(); if (!target) return BAN::Error::from_errno(ECONNREFUSED); } @@ -150,7 +152,7 @@ namespace Kernel { auto& target_info = target->m_info.get(); { - LockGuard _(target_info.pending_lock); + SpinLockGuard _(target_info.pending_lock); if (target_info.pending_connections.size() < target_info.pending_connections.capacity()) { MUST(target_info.pending_connections.push(this)); @@ -205,7 +207,7 @@ namespace Kernel O_RDWR )); - LockGuard _(s_bound_socket_lock); + SpinLockGuard _(s_bound_socket_lock); ASSERT(!s_bound_sockets.contains(file.canonical_path)); TRY(s_bound_sockets.emplace(file.canonical_path, TRY(get_weak_ptr()))); m_bound_path = BAN::move(file.canonical_path); @@ -229,12 +231,12 @@ namespace Kernel BAN::ErrorOr UnixDomainSocket::add_packet(BAN::ConstByteSpan packet) { - LockGuard _(m_lock); - + auto state = m_packet_lock.lock(); while (m_packet_sizes.full() || m_packet_size_total + packet.size() > s_packet_buffer_size) { - LockFreeGuard _(m_lock); + m_packet_lock.unlock(state); TRY(Thread::current().block_or_eintr_indefinite(m_packet_semaphore)); + state = m_packet_lock.lock(); } uint8_t* packet_buffer = reinterpret_cast(m_packet_buffer->vaddr() + m_packet_size_total); @@ -245,6 +247,7 @@ namespace Kernel m_packet_sizes.push(packet.size()); m_packet_semaphore.unblock(); + m_packet_lock.unlock(state); return {}; } @@ -318,10 +321,11 @@ namespace Kernel canonical_path = BAN::move(file.canonical_path); } - LockGuard _(s_bound_socket_lock); - if (!s_bound_sockets.contains(canonical_path)) + SpinLockGuard _(s_bound_socket_lock); + auto it = s_bound_sockets.find(canonical_path); + if (it == s_bound_sockets.end()) return BAN::Error::from_errno(EDESTADDRREQ); - auto target = s_bound_sockets[canonical_path].lock(); + auto target = it->value.lock(); if (!target) return BAN::Error::from_errno(EDESTADDRREQ); TRY(target->add_packet(message)); @@ -338,10 +342,12 @@ namespace Kernel return BAN::Error::from_errno(ENOTCONN); } + auto state = m_packet_lock.lock(); while (m_packet_size_total == 0) { - LockFreeGuard _(m_lock); + m_packet_lock.unlock(state); TRY(Thread::current().block_or_eintr_indefinite(m_packet_semaphore)); + state = m_packet_lock.lock(); } uint8_t* packet_buffer = reinterpret_cast(m_packet_buffer->vaddr()); @@ -360,6 +366,7 @@ namespace Kernel m_packet_size_total -= nread; m_packet_semaphore.unblock(); + m_packet_lock.unlock(state); return nread; } diff --git a/kernel/kernel/Process.cpp b/kernel/kernel/Process.cpp index e0f2e6c5..e7421543 100644 --- a/kernel/kernel/Process.cpp +++ b/kernel/kernel/Process.cpp @@ -7,11 +7,10 @@ #include #include #include -#include +#include #include #include #include -#include #include #include #include @@ -33,9 +32,9 @@ namespace Kernel static BAN::Vector s_processes; static RecursiveSpinLock s_process_lock; - void Process::for_each_process(const BAN::Function& callback) + static void for_each_process(const BAN::Function& callback) { - LockGuard _(s_process_lock); + SpinLockGuard _(s_process_lock); for (auto* process : s_processes) { @@ -46,9 +45,9 @@ namespace Kernel } } - void Process::for_each_process_in_session(pid_t sid, const BAN::Function& callback) + static void for_each_process_in_session(pid_t sid, const BAN::Function& callback) { - LockGuard _(s_process_lock); + SpinLockGuard _(s_process_lock); for (auto* process : s_processes) { @@ -90,9 +89,10 @@ namespace Kernel void Process::register_to_scheduler() { - s_process_lock.lock(); - MUST(s_processes.push_back(this)); - s_process_lock.unlock(); + { + SpinLockGuard _(s_process_lock); + MUST(s_processes.push_back(this)); + } for (auto* thread : m_threads) MUST(Scheduler::get().add_thread(thread)); } @@ -138,8 +138,6 @@ namespace Kernel char** argv = nullptr; { - PageTableScope _(process->page_table()); - size_t needed_bytes = sizeof(char*) * 2 + path.size() + 1; if (auto rem = needed_bytes % PAGE_SIZE) needed_bytes += PAGE_SIZE - rem; @@ -196,14 +194,14 @@ namespace Kernel void Process::add_thread(Thread* thread) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); MUST(m_threads.push_back(thread)); } void Process::cleanup_function() { { - LockGuard _(s_process_lock); + SpinLockGuard _(s_process_lock); for (size_t i = 0; i < s_processes.size(); i++) if (s_processes[i] == this) s_processes.remove(i); @@ -217,7 +215,7 @@ namespace Kernel while (m_exit_status.waiting > 0) Scheduler::get().reschedule(); - m_lock.lock(); + m_process_lock.lock(); m_open_file_descriptors.close_all(); @@ -256,7 +254,7 @@ namespace Kernel void Process::exit(int status, int signal) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); m_exit_status.exit_code = __WGENEXITCODE(status, signal); for (auto* thread : m_threads) if (thread != &Thread::current()) @@ -277,7 +275,7 @@ namespace Kernel meminfo.phys_pages = 0; { - LockGuard _(m_lock); + LockGuard _(m_process_lock); for (auto* thread : m_threads) { meminfo.virt_pages += thread->virtual_page_count(); @@ -326,13 +324,13 @@ namespace Kernel size_t Process::proc_cmdline(off_t offset, BAN::ByteSpan buffer) const { - LockGuard _(m_lock); + LockGuard _(m_process_lock); return read_from_vec_of_str(m_cmdline, offset, buffer); } size_t Process::proc_environ(off_t offset, BAN::ByteSpan buffer) const { - LockGuard _(m_lock); + LockGuard _(m_process_lock); return read_from_vec_of_str(m_environ, offset, buffer); } @@ -345,7 +343,7 @@ namespace Kernel BAN::ErrorOr Process::sys_gettermios(::termios* termios) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(validate_pointer_access(termios, sizeof(::termios))); @@ -364,7 +362,7 @@ namespace Kernel BAN::ErrorOr Process::sys_settermios(const ::termios* termios) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(validate_pointer_access(termios, sizeof(::termios))); @@ -403,7 +401,7 @@ namespace Kernel { auto page_table = BAN::UniqPtr::adopt(TRY(PageTable::create_userspace())); - LockGuard _(m_lock); + LockGuard _(m_process_lock); BAN::String working_directory; TRY(working_directory.append(m_working_directory)); @@ -443,7 +441,7 @@ namespace Kernel { // NOTE: We scope everything for automatic deletion { - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(validate_string_access(path)); auto loadable_elf = TRY(load_elf_for_exec(m_credentials, path, m_working_directory, page_table())); @@ -579,7 +577,7 @@ namespace Kernel BAN::ErrorOr Process::sys_wait(pid_t pid, int* stat_loc, int options) { { - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(validate_pointer_access(stat_loc, sizeof(int))); } @@ -612,7 +610,7 @@ namespace Kernel BAN::ErrorOr Process::sys_nanosleep(const timespec* rqtp, timespec* rmtp) { { - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(validate_pointer_access(rqtp, sizeof(timespec))); if (rmtp) TRY(validate_pointer_access(rmtp, sizeof(timespec))); @@ -654,7 +652,7 @@ namespace Kernel return BAN::Error::from_errno(ENOTSUP); } - LockGuard _(m_lock); + LockGuard _(m_process_lock); auto absolute_path = TRY(absolute_path_of(path)); @@ -683,7 +681,7 @@ namespace Kernel { ASSERT(&Process::current() == this); - LockGuard _(m_lock); + LockGuard _(m_process_lock); if (Thread::current().stack().contains(address)) { @@ -711,13 +709,13 @@ namespace Kernel BAN::ErrorOr Process::open_inode(BAN::RefPtr inode, int flags) { ASSERT(inode); - LockGuard _(m_lock); + LockGuard _(m_process_lock); return TRY(m_open_file_descriptors.open(inode, flags)); } BAN::ErrorOr Process::open_file(BAN::StringView path, int flags, mode_t mode) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); BAN::String absolute_path = TRY(absolute_path_of(path)); @@ -750,14 +748,14 @@ namespace Kernel BAN::ErrorOr Process::sys_open(const char* path, int flags, mode_t mode) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(validate_string_access(path)); return open_file(path, flags, mode); } BAN::ErrorOr Process::sys_openat(int fd, const char* path, int flags, mode_t mode) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(validate_string_access(path)); @@ -773,28 +771,28 @@ namespace Kernel BAN::ErrorOr Process::sys_close(int fd) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(m_open_file_descriptors.close(fd)); return 0; } BAN::ErrorOr Process::sys_read(int fd, void* buffer, size_t count) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(validate_pointer_access(buffer, count)); return TRY(m_open_file_descriptors.read(fd, BAN::ByteSpan((uint8_t*)buffer, count))); } BAN::ErrorOr Process::sys_write(int fd, const void* buffer, size_t count) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(validate_pointer_access(buffer, count)); return TRY(m_open_file_descriptors.write(fd, BAN::ByteSpan((uint8_t*)buffer, count))); } BAN::ErrorOr Process::sys_create(const char* path, mode_t mode) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(validate_string_access(path)); TRY(create_file_or_dir(path, mode)); return 0; @@ -802,7 +800,7 @@ namespace Kernel BAN::ErrorOr Process::sys_create_dir(const char* path, mode_t mode) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(validate_string_access(path)); BAN::StringView path_sv(path); if (!path_sv.empty() && path_sv.back() == '/') @@ -813,7 +811,7 @@ namespace Kernel BAN::ErrorOr Process::sys_unlink(const char* path) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(validate_string_access(path)); auto absolute_path = TRY(absolute_path_of(path)); @@ -846,7 +844,7 @@ namespace Kernel BAN::ErrorOr Process::sys_readlink(const char* path, char* buffer, size_t bufsize) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(validate_string_access(path)); TRY(validate_pointer_access(buffer, bufsize)); @@ -857,7 +855,7 @@ namespace Kernel BAN::ErrorOr Process::sys_readlinkat(int fd, const char* path, char* buffer, size_t bufsize) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(validate_string_access(path)); TRY(validate_pointer_access(buffer, bufsize)); @@ -874,7 +872,7 @@ namespace Kernel BAN::ErrorOr Process::sys_pread(int fd, void* buffer, size_t count, off_t offset) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(validate_pointer_access(buffer, count)); auto inode = TRY(m_open_file_descriptors.inode_of(fd)); return TRY(inode->read(offset, { (uint8_t*)buffer, count })); @@ -885,7 +883,7 @@ namespace Kernel if (mode & S_IFMASK) return BAN::Error::from_errno(EINVAL); - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(validate_string_access(path)); auto absolute_path = TRY(absolute_path_of(path)); @@ -897,7 +895,7 @@ namespace Kernel BAN::ErrorOr Process::sys_chown(const char* path, uid_t uid, gid_t gid) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(validate_string_access(path)); auto absolute_path = TRY(absolute_path_of(path)); @@ -909,7 +907,7 @@ namespace Kernel BAN::ErrorOr Process::sys_socket(int domain, int type, int protocol) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); return TRY(m_open_file_descriptors.socket(domain, type, protocol)); } @@ -920,7 +918,7 @@ namespace Kernel if (!address && address_len) return BAN::Error::from_errno(EINVAL); - LockGuard _(m_lock); + LockGuard _(m_process_lock); if (address) { TRY(validate_pointer_access(address_len, sizeof(*address_len))); @@ -936,7 +934,7 @@ namespace Kernel BAN::ErrorOr Process::sys_bind(int socket, const sockaddr* address, socklen_t address_len) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(validate_pointer_access(address, address_len)); auto inode = TRY(m_open_file_descriptors.inode_of(socket)); @@ -949,7 +947,7 @@ namespace Kernel BAN::ErrorOr Process::sys_connect(int socket, const sockaddr* address, socklen_t address_len) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(validate_pointer_access(address, address_len)); auto inode = TRY(m_open_file_descriptors.inode_of(socket)); @@ -962,7 +960,7 @@ namespace Kernel BAN::ErrorOr Process::sys_listen(int socket, int backlog) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); auto inode = TRY(m_open_file_descriptors.inode_of(socket)); if (!inode->mode().ifsock()) @@ -974,7 +972,7 @@ namespace Kernel BAN::ErrorOr Process::sys_sendto(const sys_sendto_t* arguments) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(validate_pointer_access(arguments, sizeof(sys_sendto_t))); TRY(validate_pointer_access(arguments->message, arguments->length)); TRY(validate_pointer_access(arguments->dest_addr, arguments->dest_len)); @@ -994,7 +992,7 @@ namespace Kernel if (!arguments->address && arguments->address_len) return BAN::Error::from_errno(EINVAL); - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(validate_pointer_access(arguments, sizeof(sys_recvfrom_t))); TRY(validate_pointer_access(arguments->buffer, arguments->length)); if (arguments->address) @@ -1013,14 +1011,14 @@ namespace Kernel BAN::ErrorOr Process::sys_ioctl(int fildes, int request, void* arg) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); auto inode = TRY(m_open_file_descriptors.inode_of(fildes)); return TRY(inode->ioctl(request, arg)); } BAN::ErrorOr Process::sys_pselect(sys_pselect_t* arguments) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(validate_pointer_access(arguments, sizeof(sys_pselect_t))); if (arguments->readfds) @@ -1089,7 +1087,7 @@ namespace Kernel if (set_bits > 0) break; - LockFreeGuard free(m_lock); + LockFreeGuard free(m_process_lock); SystemTimer::get().sleep(1); } @@ -1115,7 +1113,7 @@ namespace Kernel BAN::ErrorOr Process::sys_pipe(int fildes[2]) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(validate_pointer_access(fildes, sizeof(int) * 2)); TRY(m_open_file_descriptors.pipe(fildes)); return 0; @@ -1123,32 +1121,32 @@ namespace Kernel BAN::ErrorOr Process::sys_dup(int fildes) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); return TRY(m_open_file_descriptors.dup(fildes)); } BAN::ErrorOr Process::sys_dup2(int fildes, int fildes2) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); return TRY(m_open_file_descriptors.dup2(fildes, fildes2)); } BAN::ErrorOr Process::sys_fcntl(int fildes, int cmd, int extra) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); return TRY(m_open_file_descriptors.fcntl(fildes, cmd, extra)); } BAN::ErrorOr Process::sys_seek(int fd, off_t offset, int whence) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(m_open_file_descriptors.seek(fd, offset, whence)); return 0; } BAN::ErrorOr Process::sys_tell(int fd) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); return TRY(m_open_file_descriptors.tell(fd)); } @@ -1156,7 +1154,7 @@ namespace Kernel { BAN::String absolute_source, absolute_target; { - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(absolute_source.append(TRY(absolute_path_of(source)))); TRY(absolute_target.append(TRY(absolute_path_of(target)))); } @@ -1166,7 +1164,7 @@ namespace Kernel BAN::ErrorOr Process::sys_fstat(int fd, struct stat* buf) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(validate_pointer_access(buf, sizeof(struct stat))); TRY(m_open_file_descriptors.fstat(fd, buf)); return 0; @@ -1174,7 +1172,7 @@ namespace Kernel BAN::ErrorOr Process::sys_fstatat(int fd, const char* path, struct stat* buf, int flag) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(validate_pointer_access(buf, sizeof(struct stat))); TRY(m_open_file_descriptors.fstatat(fd, path, buf, flag)); return 0; @@ -1182,7 +1180,7 @@ namespace Kernel BAN::ErrorOr Process::sys_stat(const char* path, struct stat* buf, int flag) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(validate_pointer_access(buf, sizeof(struct stat))); TRY(m_open_file_descriptors.stat(TRY(absolute_path_of(path)), buf, flag)); return 0; @@ -1235,7 +1233,7 @@ namespace Kernel BAN::ErrorOr Process::sys_readdir(int fd, DirectoryEntryList* list, size_t list_size) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(validate_pointer_access(list, list_size)); TRY(m_open_file_descriptors.read_dir_entries(fd, list, list_size)); return 0; @@ -1246,7 +1244,7 @@ namespace Kernel BAN::String absolute_path; { - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(validate_string_access(path)); absolute_path = TRY(absolute_path_of(path)); } @@ -1255,7 +1253,7 @@ namespace Kernel if (!file.inode->mode().ifdir()) return BAN::Error::from_errno(ENOTDIR); - LockGuard _(m_lock); + LockGuard _(m_process_lock); m_working_directory = BAN::move(file.canonical_path); return 0; @@ -1263,7 +1261,7 @@ namespace Kernel BAN::ErrorOr Process::sys_getpwd(char* buffer, size_t size) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(validate_pointer_access(buffer, size)); @@ -1279,7 +1277,7 @@ namespace Kernel BAN::ErrorOr Process::sys_mmap(const sys_mmap_t* args) { { - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(validate_pointer_access(args, sizeof(sys_mmap_t))); } @@ -1320,7 +1318,7 @@ namespace Kernel region_type, page_flags )); - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(m_mapped_regions.push_back(BAN::move(region))); return m_mapped_regions.back()->vaddr(); } @@ -1328,7 +1326,7 @@ namespace Kernel if (args->addr != nullptr) return BAN::Error::from_errno(ENOTSUP); - LockGuard _(m_lock); + LockGuard _(m_process_lock); auto inode = TRY(m_open_file_descriptors.inode_of(args->fildes)); @@ -1376,7 +1374,7 @@ namespace Kernel if (vaddr % PAGE_SIZE != 0) return BAN::Error::from_errno(EINVAL); - LockGuard _(m_lock); + LockGuard _(m_process_lock); // FIXME: We should only map partial regions for (size_t i = 0; i < m_mapped_regions.size(); i++) @@ -1395,7 +1393,7 @@ namespace Kernel if (vaddr % PAGE_SIZE != 0) return BAN::Error::from_errno(EINVAL); - LockGuard _(m_lock); + LockGuard _(m_process_lock); for (auto& mapped_region : m_mapped_regions) if (mapped_region->overlaps(vaddr, len)) @@ -1406,7 +1404,7 @@ namespace Kernel BAN::ErrorOr Process::sys_tty_ctrl(int fildes, int command, int flags) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); auto inode = TRY(m_open_file_descriptors.inode_of(fildes)); if (!inode->is_tty()) @@ -1419,7 +1417,7 @@ namespace Kernel BAN::ErrorOr Process::sys_termid(char* buffer) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(validate_string_access(buffer)); @@ -1440,7 +1438,7 @@ namespace Kernel BAN::ErrorOr Process::sys_clock_gettime(clockid_t clock_id, timespec* tp) { { - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(validate_pointer_access(tp, sizeof(timespec))); } @@ -1465,7 +1463,7 @@ namespace Kernel BAN::ErrorOr Process::sys_load_keymap(const char* path) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(validate_string_access(path)); if (!m_credentials.is_superuser()) @@ -1482,7 +1480,7 @@ namespace Kernel return BAN::Error::from_errno(EINVAL); { - LockGuard _(m_lock); + LockGuard _(m_process_lock); TRY(validate_pointer_access((void*)handler, sizeof(handler))); } @@ -1532,7 +1530,7 @@ namespace Kernel BAN::ErrorOr Process::sys_tcsetpgrp(int fd, pid_t pgrp) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); if (!m_controlling_terminal) return BAN::Error::from_errno(ENOTTY); @@ -1568,7 +1566,7 @@ namespace Kernel if (uid < 0 || uid >= 1'000'000'000) return BAN::Error::from_errno(EINVAL); - LockGuard _(m_lock); + LockGuard _(m_process_lock); // If the process has appropriate privileges, setuid() shall set the real user ID, effective user ID, and the saved // set-user-ID of the calling process to uid. @@ -1598,7 +1596,7 @@ namespace Kernel if (gid < 0 || gid >= 1'000'000'000) return BAN::Error::from_errno(EINVAL); - LockGuard _(m_lock); + LockGuard _(m_process_lock); // If the process has appropriate privileges, setgid() shall set the real group ID, effective group ID, and the saved // set-group-ID of the calling process to gid. @@ -1626,7 +1624,7 @@ namespace Kernel if (uid < 0 || uid >= 1'000'000'000) return BAN::Error::from_errno(EINVAL); - LockGuard _(m_lock); + LockGuard _(m_process_lock); // If uid is equal to the real user ID or the saved set-user-ID, or if the process has appropriate privileges, seteuid() // shall set the effective user ID of the calling process to uid; the real user ID and saved set-user-ID shall remain unchanged. @@ -1645,7 +1643,7 @@ namespace Kernel if (gid < 0 || gid >= 1'000'000'000) return BAN::Error::from_errno(EINVAL); - LockGuard _(m_lock); + LockGuard _(m_process_lock); // If gid is equal to the real group ID or the saved set-group-ID, or if the process has appropriate privileges, setegid() // shall set the effective group ID of the calling process to gid; the real group ID, saved set-group-ID, and any @@ -1673,7 +1671,7 @@ namespace Kernel // by the ruid and euid arguments. If ruid or euid is -1, the corresponding effective or real user ID of the current // process shall be left unchanged. - LockGuard _(m_lock); + LockGuard _(m_process_lock); // A process with appropriate privileges can set either ID to any value. if (!m_credentials.is_superuser()) @@ -1721,7 +1719,7 @@ namespace Kernel // The real and effective group IDs may be set to different values in the same call. - LockGuard _(m_lock); + LockGuard _(m_process_lock); // Only a process with appropriate privileges can set the real group ID and the effective group ID to any valid value. if (!m_credentials.is_superuser()) @@ -1754,7 +1752,7 @@ namespace Kernel if (pgid < 0) return BAN::Error::from_errno(EINVAL); - LockGuard _(m_lock); + LockGuard _(m_process_lock); if (pid == 0) pid = m_pid; @@ -1819,7 +1817,7 @@ namespace Kernel BAN::ErrorOr Process::sys_getpgid(pid_t pid) { - LockGuard _(m_lock); + LockGuard _(m_process_lock); if (pid == 0 || pid == m_pid) return m_pgrp; @@ -1851,7 +1849,7 @@ namespace Kernel BAN::ErrorOr Process::absolute_path_of(BAN::StringView path) const { - LockGuard _(m_lock); + LockGuard _(m_process_lock); if (path.empty() || path == "."sv) return m_working_directory; diff --git a/kernel/kernel/Scheduler.cpp b/kernel/kernel/Scheduler.cpp index 19f9039b..93ec9805 100644 --- a/kernel/kernel/Scheduler.cpp +++ b/kernel/kernel/Scheduler.cpp @@ -35,9 +35,10 @@ namespace Kernel BAN::ErrorOr Scheduler::initialize() { ASSERT(s_instance == nullptr); - s_instance = new Scheduler(); - ASSERT(s_instance); - s_instance->m_idle_thread = TRY(Thread::create_kernel([](void*) { for (;;) asm volatile("hlt"); }, nullptr, nullptr)); + Scheduler* scheduler = new Scheduler(); + ASSERT(scheduler); + scheduler->m_idle_thread = TRY(Thread::create_kernel([](void*) { for (;;) asm volatile("hlt"); }, nullptr, nullptr)); + s_instance = scheduler; return {}; } diff --git a/kernel/kernel/SpinLock.cpp b/kernel/kernel/SpinLock.cpp deleted file mode 100644 index 6a56a46f..00000000 --- a/kernel/kernel/SpinLock.cpp +++ /dev/null @@ -1,86 +0,0 @@ -#include -#include - -namespace Kernel -{ - - void SpinLock::lock() - { - pid_t tid = Scheduler::current_tid(); - ASSERT(tid != m_locker); - while (!m_locker.compare_exchange(-1, tid)) - Scheduler::get().reschedule(); - } - - void SpinLock::unlock() - { - ASSERT(m_locker == Scheduler::current_tid()); - m_locker = -1; - } - - bool SpinLock::is_locked() const - { - return m_locker != -1; - } - - void RecursiveSpinLock::lock() - { - pid_t tid = Scheduler::current_tid(); - if (m_locker != tid) - { - while (!m_locker.compare_exchange(-1, tid)) - Scheduler::get().reschedule(); - ASSERT(m_lock_depth == 0); - } - m_lock_depth++; - } - - void RecursiveSpinLock::unlock() - { - ASSERT(m_lock_depth > 0); - ASSERT(m_locker == Scheduler::current_tid()); - if (--m_lock_depth == 0) - m_locker = -1; - } - - bool RecursiveSpinLock::is_locked() const - { - return m_locker != -1; - } - - void RecursivePrioritySpinLock::lock() - { - pid_t tid = Scheduler::current_tid(); - - bool has_priority = !Thread::current().is_userspace(); - if (has_priority) - m_queue_length++; - - if (m_locker != tid) - { - while (!((has_priority || m_queue_length == 0) && m_locker.compare_exchange(-1, tid))) - Scheduler::get().reschedule(); - ASSERT(m_lock_depth == 0); - } - m_lock_depth++; - } - - void RecursivePrioritySpinLock::unlock() - { - ASSERT(m_lock_depth > 0); - ASSERT(m_locker == Scheduler::current_tid()); - - bool has_priority = !Thread::current().is_userspace(); - if (has_priority) - m_queue_length--; - - if (--m_lock_depth == 0) - m_locker = -1; - } - - bool RecursivePrioritySpinLock::is_locked() const - { - return m_locker != -1; - } - -} diff --git a/kernel/kernel/Storage/ATA/ATABus.cpp b/kernel/kernel/Storage/ATA/ATABus.cpp index 157ca480..7ff38820 100644 --- a/kernel/kernel/Storage/ATA/ATABus.cpp +++ b/kernel/kernel/Storage/ATA/ATABus.cpp @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include #include #include @@ -261,7 +261,7 @@ namespace Kernel if (lba + sector_count > device.sector_count()) return BAN::Error::from_error_code(ErrorCode::Storage_Boundaries); - LockGuard _(m_lock); + LockGuard _(m_mutex); if (lba < (1 << 28)) { @@ -298,7 +298,7 @@ namespace Kernel if (lba + sector_count > device.sector_count()) return BAN::Error::from_error_code(ErrorCode::Storage_Boundaries); - LockGuard _(m_lock); + LockGuard _(m_mutex); if (lba < (1 << 28)) { diff --git a/kernel/kernel/Storage/DiskCache.cpp b/kernel/kernel/Storage/DiskCache.cpp index 3e682b74..b8302ee9 100644 --- a/kernel/kernel/Storage/DiskCache.cpp +++ b/kernel/kernel/Storage/DiskCache.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include #include @@ -32,10 +32,6 @@ namespace Kernel uint64_t page_cache_offset = sector % sectors_per_page; uint64_t page_cache_start = sector - page_cache_offset; - PageTable& page_table = PageTable::current(); - LockGuard page_table_locker(page_table); - ASSERT(page_table.is_page_free(0)); - for (auto& cache : m_cache) { if (cache.first_sector < page_cache_start) @@ -46,10 +42,9 @@ namespace Kernel if (!(cache.sector_mask & (1 << page_cache_offset))) continue; - CriticalScope _; - PageTable::map_fast_page(cache.paddr); - memcpy(buffer.data(), PageTable::fast_page_as_ptr(page_cache_offset * m_sector_size), m_sector_size); - PageTable::unmap_fast_page(); + PageTable::with_fast_page(cache.paddr, [&] { + memcpy(buffer.data(), PageTable::fast_page_as_ptr(page_cache_offset * m_sector_size), m_sector_size); + }); return true; } @@ -64,10 +59,6 @@ namespace Kernel uint64_t page_cache_offset = sector % sectors_per_page; uint64_t page_cache_start = sector - page_cache_offset; - PageTable& page_table = PageTable::current(); - LockGuard page_table_locker(page_table); - ASSERT(page_table.is_page_free(0)); - size_t index = 0; // Search the cache if the have this sector in memory @@ -80,12 +71,9 @@ namespace Kernel if (cache.first_sector > page_cache_start) break; - { - CriticalScope _; - PageTable::map_fast_page(cache.paddr); + PageTable::with_fast_page(cache.paddr, [&] { memcpy(PageTable::fast_page_as_ptr(page_cache_offset * m_sector_size), buffer.data(), m_sector_size); - PageTable::unmap_fast_page(); - } + }); cache.sector_mask |= 1 << page_cache_offset; if (dirty) @@ -111,12 +99,9 @@ namespace Kernel return ret.error(); } - { - CriticalScope _; - PageTable::map_fast_page(cache.paddr); + PageTable::with_fast_page(cache.paddr, [&] { memcpy(PageTable::fast_page_as_ptr(page_cache_offset * m_sector_size), buffer.data(), m_sector_size); - PageTable::unmap_fast_page(); - } + }); return {}; } @@ -128,12 +113,9 @@ namespace Kernel if (cache.dirty_mask == 0) continue; - { - CriticalScope _; - PageTable::map_fast_page(cache.paddr); + PageTable::with_fast_page(cache.paddr, [&] { memcpy(m_sync_cache.data(), PageTable::fast_page_as_ptr(), PAGE_SIZE); - PageTable::unmap_fast_page(); - } + }); uint8_t sector_start = 0; uint8_t sector_count = 0; diff --git a/kernel/kernel/Storage/NVMe/Queue.cpp b/kernel/kernel/Storage/NVMe/Queue.cpp index 1b0395e7..f841cc24 100644 --- a/kernel/kernel/Storage/NVMe/Queue.cpp +++ b/kernel/kernel/Storage/NVMe/Queue.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -44,7 +44,7 @@ namespace Kernel uint16_t NVMeQueue::submit_command(NVMe::SubmissionQueueEntry& sqe) { - LockGuard _(m_lock); + LockGuard _(m_mutex); ASSERT(m_done == false); m_status = 0; diff --git a/kernel/kernel/Storage/StorageDevice.cpp b/kernel/kernel/Storage/StorageDevice.cpp index 746403dc..3e1e4d79 100644 --- a/kernel/kernel/Storage/StorageDevice.cpp +++ b/kernel/kernel/Storage/StorageDevice.cpp @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include #include @@ -150,7 +150,7 @@ namespace Kernel BAN::Vector lba1; TRY(lba1.resize(sector_size())); - TRY(read_sectors(1, 1, lba1.span())); + TRY(read_sectors(1, 1, BAN::ByteSpan { lba1.span() })); const GPTHeader& header = *(const GPTHeader*)lba1.data(); if (!is_valid_gpt_header(header, sector_size())) @@ -165,7 +165,7 @@ namespace Kernel BAN::Vector entry_array; TRY(entry_array.resize(size)); - TRY(read_sectors(header.partition_entry_lba, size / sector_size(), entry_array.span())); + TRY(read_sectors(header.partition_entry_lba, size / sector_size(), BAN::ByteSpan { entry_array.span() })); if (!is_valid_gpt_crc32(header, lba1, entry_array)) return BAN::Error::from_error_code(ErrorCode::Storage_GPTHeader); @@ -207,14 +207,14 @@ namespace Kernel void StorageDevice::add_disk_cache() { - LockGuard _(m_lock); + LockGuard _(m_mutex); ASSERT(!m_disk_cache.has_value()); m_disk_cache.emplace(sector_size(), *this); } BAN::ErrorOr StorageDevice::sync_disk_cache() { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (m_disk_cache.has_value()) TRY(m_disk_cache->sync()); return {}; @@ -225,14 +225,14 @@ namespace Kernel ASSERT(buffer.size() >= sector_count * sector_size()); { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (!m_disk_cache.has_value()) return read_sectors_impl(lba, sector_count, buffer); } for (uint64_t offset = 0; offset < sector_count; offset++) { - LockGuard _(m_lock); + LockGuard _(m_mutex); auto sector_buffer = buffer.slice(offset * sector_size(), sector_size()); if (m_disk_cache->read_from_cache(lba + offset, sector_buffer)) continue; @@ -248,14 +248,14 @@ namespace Kernel ASSERT(buffer.size() >= sector_count * sector_size()); { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (!m_disk_cache.has_value()) return write_sectors_impl(lba, sector_count, buffer); } for (uint8_t offset = 0; offset < sector_count; offset++) { - LockGuard _(m_lock); + LockGuard _(m_mutex); auto sector_buffer = buffer.slice(offset * sector_size(), sector_size()); if (m_disk_cache->write_to_cache(lba + offset, sector_buffer, true).is_error()) TRY(write_sectors_impl(lba + offset, 1, sector_buffer)); diff --git a/kernel/kernel/Terminal/TTY.cpp b/kernel/kernel/Terminal/TTY.cpp index 14e1d09c..3a621cee 100644 --- a/kernel/kernel/Terminal/TTY.cpp +++ b/kernel/kernel/Terminal/TTY.cpp @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include @@ -122,7 +122,7 @@ namespace Kernel void TTY::on_key_event(Input::KeyEvent event) { - LockGuard _(m_lock); + LockGuard _(m_mutex); if (event.released()) return; @@ -205,7 +205,7 @@ namespace Kernel if (ch == 0) return; - LockGuard _(m_lock); + LockGuard _(m_mutex); // ^C if (ch == '\x03') @@ -310,24 +310,17 @@ namespace Kernel void TTY::putchar(uint8_t ch) { + SpinLockGuard _(m_write_lock); if (m_tty_ctrl.draw_graphics) putchar_impl(ch); } BAN::ErrorOr TTY::read_impl(off_t, BAN::ByteSpan buffer) { - LockGuard _(m_lock); while (!m_output.flush) { - // FIXME: this is very hacky way to unlock lock temporarily - uint32_t depth = m_lock.lock_depth(); - for (uint32_t i = 0; i < depth; i++) - m_lock.unlock(); - auto eintr = Thread::current().block_or_eintr_indefinite(m_output.semaphore); - for (uint32_t i = 0; i < depth; i++) - m_lock.lock(); - if (eintr.is_error()) - return eintr.release_error(); + LockFreeGuard _(m_mutex); + TRY(Thread::current().block_or_eintr_indefinite(m_output.semaphore)); } if (m_output.bytes == 0) @@ -352,7 +345,7 @@ namespace Kernel BAN::ErrorOr TTY::write_impl(off_t, BAN::ConstByteSpan buffer) { - LockGuard _(m_lock); + SpinLockGuard _(m_write_lock); for (size_t i = 0; i < buffer.size(); i++) putchar(buffer[i]); return buffer.size(); @@ -361,7 +354,7 @@ namespace Kernel void TTY::putchar_current(uint8_t ch) { ASSERT(s_tty); - LockGuard _(s_tty->m_lock); + SpinLockGuard _(s_tty->m_write_lock); s_tty->putchar(ch); } diff --git a/kernel/kernel/Terminal/VirtualTTY.cpp b/kernel/kernel/Terminal/VirtualTTY.cpp index 59b5806c..0fb90b7c 100644 --- a/kernel/kernel/Terminal/VirtualTTY.cpp +++ b/kernel/kernel/Terminal/VirtualTTY.cpp @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include @@ -306,8 +306,6 @@ namespace Kernel void VirtualTTY::putchar_impl(uint8_t ch) { - ASSERT(m_lock.is_locked()); - uint32_t codepoint = ch; switch (m_state) diff --git a/kernel/kernel/Thread.cpp b/kernel/kernel/Thread.cpp index 1648cb7d..f21be637 100644 --- a/kernel/kernel/Thread.cpp +++ b/kernel/kernel/Thread.cpp @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #include