diff --git a/kernel/include/kernel/Process.h b/kernel/include/kernel/Process.h index d619d0c0..1df6c4c5 100644 --- a/kernel/include/kernel/Process.h +++ b/kernel/include/kernel/Process.h @@ -346,6 +346,19 @@ namespace Kernel vaddr_t m_shared_page_vaddr { 0 }; + struct futex_t + { + ThreadBlocker blocker; + uint32_t waiters { 0 }; + uint32_t to_wakeup { 0 }; + }; + + static BAN::HashMap> s_futexes; + static Mutex s_futex_lock; + + BAN::HashMap> m_futexes; + Mutex m_futex_lock; + BAN::Vector m_threads; struct pthread_info_t diff --git a/kernel/kernel/Process.cpp b/kernel/kernel/Process.cpp index 87df969d..a833ba9d 100644 --- a/kernel/kernel/Process.cpp +++ b/kernel/kernel/Process.cpp @@ -38,14 +38,8 @@ namespace Kernel static BAN::Vector s_processes; static RecursiveSpinLock s_process_lock; - struct futex_t - { - ThreadBlocker blocker; - uint32_t waiters { 0 }; - uint32_t to_wakeup { 0 }; - }; - static BAN::HashMap> s_futexes; - static Mutex s_futex_lock; + BAN::HashMap> Process::s_futexes; + Mutex Process::s_futex_lock; static void for_each_process(const BAN::Function& callback) { @@ -3107,18 +3101,52 @@ namespace Kernel return BAN::Error::from_errno(EINVAL); const bool is_realtime = (op & FUTEX_REALTIME); + const bool is_private = (op & FUTEX_PRIVATE); op &= ~(FUTEX_PRIVATE | FUTEX_REALTIME); - // TODO: possibly optimize private futexes? - - LockGuard _(s_futex_lock); - auto* buffer_region = TRY(validate_and_pin_pointer_access(addr, sizeof(uint32_t), false)); BAN::ScopeGuard pin_guard([&] { if (buffer_region) buffer_region->unpin(); }); const paddr_t paddr = m_page_table->physical_address_of(vaddr & PAGE_ADDR_MASK) | (vaddr & ~PAGE_ADDR_MASK); ASSERT(paddr != 0); + switch (op) + { + case FUTEX_WAIT: + break; + case FUTEX_WAKE: + abstime = nullptr; + break; + default: + return BAN::Error::from_errno(ENOSYS); + } + + const uint64_t wake_time_ns = + TRY([abstime, is_realtime, this]() -> BAN::ErrorOr + { + if (abstime == nullptr) + return BAN::numeric_limits::max(); + const uint64_t abs_ns = + TRY([abstime, this]() -> BAN::ErrorOr + { + LockGuard _(m_process_lock); + TRY(validate_pointer_access(abstime, sizeof(*abstime), false)); + return abstime->tv_sec * 1'000'000'000 + abstime->tv_nsec; + }()); + if (!is_realtime) + return abs_ns; + const auto realtime = SystemTimer::get().real_time(); + const uint64_t real_ns = realtime.tv_sec * 1'000'000'000 + realtime.tv_nsec; + if (abs_ns <= real_ns) + return BAN::Error::from_errno(ETIMEDOUT); + return SystemTimer::get().ns_since_boot() + (abs_ns - real_ns); + }()); + + auto& futex_lock = is_private ? m_futex_lock : s_futex_lock; + auto& futexes = is_private ? m_futexes : s_futexes; + + LockGuard _(futex_lock); + switch (op) { case FUTEX_WAIT: @@ -3126,36 +3154,20 @@ namespace Kernel if (BAN::atomic_load(*addr) != val) return BAN::Error::from_errno(EAGAIN); - const uint64_t wake_time_ns = - TRY([abstime, is_realtime, this]() -> BAN::ErrorOr - { - if (abstime == nullptr) - return BAN::numeric_limits::max(); - TRY(validate_pointer_access(abstime, sizeof(*abstime), false)); - const uint64_t abs_ns = abstime->tv_sec * 1'000'000'000 + abstime->tv_nsec; - if (!is_realtime) - return abs_ns; - const auto realtime = SystemTimer::get().real_time(); - const uint64_t real_ns = realtime.tv_sec * 1'000'000'000 + realtime.tv_nsec; - if (abs_ns <= real_ns) - return BAN::Error::from_errno(ETIMEDOUT); - return SystemTimer::get().ns_since_boot() + (abs_ns - real_ns); - }()); - - auto it = s_futexes.find(paddr); - if (it == s_futexes.end()) - it = TRY(s_futexes.emplace(paddr, TRY(BAN::UniqPtr::create()))); + auto it = futexes.find(paddr); + if (it == futexes.end()) + it = TRY(futexes.emplace(paddr, TRY(BAN::UniqPtr::create()))); futex_t* const futex = it->value.ptr(); futex->waiters++; - BAN::ScopeGuard _([futex, paddr] { + BAN::ScopeGuard _([&futexes, futex, paddr] { if (--futex->waiters == 0) - s_futexes.remove(paddr); + futexes.remove(paddr); }); for (;;) { - TRY(Thread::current().block_or_eintr_or_waketime_ns(futex->blocker, wake_time_ns, true, &s_futex_lock)); + TRY(Thread::current().block_or_eintr_or_waketime_ns(futex->blocker, wake_time_ns, true, &futex_lock)); if (BAN::atomic_load(*addr) == val || futex->to_wakeup == 0) continue; futex->to_wakeup--; @@ -3164,8 +3176,8 @@ namespace Kernel } case FUTEX_WAKE: { - auto it = s_futexes.find(paddr); - if (it == s_futexes.end()) + auto it = futexes.find(paddr); + if (it == futexes.end()) return 0; futex_t* const futex = it->value.ptr(); @@ -3178,8 +3190,6 @@ namespace Kernel futex->blocker.unblock(); return 0; } - default: - return BAN::Error::from_errno(ENOSYS); } ASSERT_NOT_REACHED(); diff --git a/userspace/libraries/LibC/pthread.cpp b/userspace/libraries/LibC/pthread.cpp index 3befd45c..5c8a0287 100644 --- a/userspace/libraries/LibC/pthread.cpp +++ b/userspace/libraries/LibC/pthread.cpp @@ -884,9 +884,11 @@ int pthread_mutex_unlock(pthread_mutex_t* mutex) mutex->lock_depth--; if (mutex->lock_depth == 0) { + const int op = FUTEX_WAKE | (mutex->attr.shared ? 0 : FUTEX_PRIVATE); + BAN::atomic_store(mutex->futex, 0, BAN::memory_order_release); if (BAN::atomic_load(mutex->waiters)) - futex(FUTEX_WAKE, &mutex->futex, 1, nullptr); + futex(op, &mutex->futex, 1, nullptr); } return 0; diff --git a/userspace/libraries/LibC/semaphore.cpp b/userspace/libraries/LibC/semaphore.cpp index 2f85aff4..a3703b0a 100644 --- a/userspace/libraries/LibC/semaphore.cpp +++ b/userspace/libraries/LibC/semaphore.cpp @@ -28,8 +28,9 @@ int sem_getvalue(sem_t* __restrict sem, int* __restrict sval) int sem_post(sem_t* sem) { const auto old = BAN::atomic_fetch_add(sem->value, 1); + const int op = FUTEX_WAKE | (sem->shared ? 0 : FUTEX_PRIVATE); if (old == 0) - futex(FUTEX_WAKE, &sem->value, 1, nullptr); + futex(op, &sem->value, 1, nullptr); return 0; }