Sinipelto
/
banan-os
forked from Bananymous/banan-os
1
0
Fork 0
Code Pull Requests Activity

Kernel: All processors use LAPIC timer when running with APIC 

This makes scheduler preemption much cleaner as bsb does not have to
send smp messages to notify other processes about timer interrupt.

Also PIT percision is now "full" 0.8 us instead of 1 ms that I was using
before.
This commit is contained in:
Bananymous 2024-07-23 02:28:52 +03:00
parent 3e0150f847
commit 539afb329a
17 changed files with 224 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
kernel/arch/i686/interrupts.S
View File

@ -113,6 +113,23 @@ asm_ipi_handler:
pop_userspace
iret
.global asm_timer_handler
asm_timer_handler:
push_userspace
load_kernel_segments
movl %esp, %ebp
subl $15, %esp
andl $0xFFFFFFF0, %esp
call cpp_timer_handler
movl %ebp, %esp
pop_userspace
iret
.macro isr n
.global isr\n
isr\n:

7
kernel/arch/x86_64/interrupts.S
View File

@ -77,6 +77,13 @@ asm_ipi_handler:
popaq
iretq
.global asm_timer_handler
asm_timer_handler:
pushaq
call cpp_timer_handler
popaq
iretq
.macro isr n
.global isr\n
isr\n:

17
kernel/include/kernel/APIC.h
View File

@ -23,6 +23,8 @@ namespace Kernel
virtual void broadcast_ipi() override;
virtual void enable() override;
void initialize_timer();
private:
uint32_t read_from_local_apic(ptrdiff_t);
void write_to_local_apic(ptrdiff_t, uint32_t);
@ -58,13 +60,14 @@ namespace Kernel
};
private:
SpinLock m_lock;
BAN::Vector<Processor> m_processors;
Kernel::paddr_t m_local_apic_paddr = 0;
Kernel::vaddr_t m_local_apic_vaddr = 0;
BAN::Vector<IOAPIC> m_io_apics;
uint8_t m_irq_overrides[0x100] {};
uint8_t m_reserved_gsis[0x100 / 8] {};
SpinLock m_lock;
BAN::Vector<Processor> m_processors;
Kernel::paddr_t m_local_apic_paddr = 0;
Kernel::vaddr_t m_local_apic_vaddr = 0;
BAN::Vector<IOAPIC> m_io_apics;
uint8_t m_irq_overrides[0x100] {};
uint8_t m_reserved_gsis[0x100 / 8] {};
uint64_t m_lapic_timer_frequency_hz { 0 };
};
}

3
kernel/include/kernel/IDT.h
View File

@ -7,9 +7,10 @@
#include <stdint.h>
constexpr uint8_t IRQ_VECTOR_BASE = 0x20;
constexpr uint8_t IRQ_VECTOR_BASE = 0x20;
constexpr uint8_t IRQ_YIELD = 32;
constexpr uint8_t IRQ_IPI = 33;
constexpr uint8_t IRQ_TIMER = 34;
namespace Kernel
{

2
kernel/include/kernel/Processor.h
View File

@ -56,8 +56,6 @@ namespace Kernel
FlushTLB,
NewThread,
UnblockThread,
// FIXME: all processors should LAPIC for their preemption
SchedulerPreemption,
};
SMPMessage* next { nullptr };
Type type;

2
kernel/include/kernel/Scheduler.h
View File

@ -110,8 +110,6 @@ namespace Kernel
class ProcessorID find_least_loaded_processor() const;
void preempt();
void handle_unblock_request(const UnblockRequest&);
void handle_new_thread_request(const NewThreadRequest&);

2
kernel/include/kernel/Timer/HPET.h
View File

@ -19,6 +19,8 @@ namespace Kernel
virtual uint64_t ns_since_boot() const override;
virtual timespec time_since_boot() const override;
virtual void pre_scheduler_sleep_ns(uint64_t) override;
virtual void handle_irq() override;
private:

7
kernel/include/kernel/Timer/PIT.h
View File

@ -16,15 +16,18 @@ namespace Kernel
virtual uint64_t ns_since_boot() const override;
virtual timespec time_since_boot() const override;
virtual void pre_scheduler_sleep_ns(uint64_t) override;
virtual void handle_irq() override;
private:
void initialize();
uint64_t read_counter() const;
uint64_t read_counter_ns() const;
private:
mutable SpinLock m_lock;
uint64_t m_system_time { 0 };
uint64_t m_system_time_ms { 0 };
};
}

12
kernel/include/kernel/Timer/Timer.h
View File

@ -16,6 +16,16 @@ namespace Kernel
virtual uint64_t ms_since_boot() const = 0;
virtual uint64_t ns_since_boot() const = 0;
virtual timespec time_since_boot() const = 0;
virtual void pre_scheduler_sleep_ns(uint64_t) = 0;
void dont_invoke_scheduler() { m_should_invoke_scheduler = false; }
protected:
bool should_invoke_scheduler() const { return m_should_invoke_scheduler; }
private:
bool m_should_invoke_scheduler { true };
};
class SystemTimer : public Timer
@ -29,6 +39,8 @@ namespace Kernel
virtual uint64_t ns_since_boot() const override;
virtual timespec time_since_boot() const override;
virtual void pre_scheduler_sleep_ns(uint64_t) override;
void sleep_ms(uint64_t ms) const { return sleep_ns(ms * 1'000'000); }
void sleep_ns(uint64_t ns) const;

58
kernel/kernel/APIC.cpp
View File

@ -17,6 +17,11 @@
#define LAPIC_ICR_LO_REG 0x300
#define LAPIC_ICR_HI_REG 0x310
#define LAPIC_TIMER_LVT 0x320
#define LAPIC_TIMER_INITIAL_REG 0x380
#define LAPIC_TIMER_CURRENT_REG 0x390
#define LAPIC_TIMER_DIVIDE_REG 0x3E0
#define IOAPIC_MAX_REDIRS 0x01
#define IOAPIC_REDIRS 0x10
@ -60,6 +65,27 @@ namespace Kernel
ICR_LO_destination_shorthand_all_excluding_self = 0b11 << 18,
};
enum TimerLVT : uint32_t
{
TimerModeOneShot = 0b00 << 17,
TimerModePeriodic = 0b01 << 17,
TimerModeTSCDeadline = 0b10 << 17,
TimerMask = 1 << 16,
};
enum TimerDivideRegister : uint32_t
{
DivideBy2 = 0b0000,
DivideBy4 = 0b0001,
DivideBy8 = 0b0010,
DivideBy16 = 0b0011,
DivideBy32 = 0b1000,
DivideBy64 = 0b1001,
DivideBy128 = 0b1010,
DivideBy1 = 0b1011,
};
struct MADT : public Kernel::ACPI::SDTHeader
{
uint32_t local_apic;
@ -223,12 +249,7 @@ namespace Kernel
void APIC::initialize_multiprocessor()
{
constexpr auto udelay =
[](uint64_t us) {
uint64_t wake_time = SystemTimer::get().ns_since_boot() + us * 1000;
while (SystemTimer::get().ns_since_boot() < wake_time)
__builtin_ia32_pause();
};
constexpr auto udelay = [](uint64_t us) { SystemTimer::get().pre_scheduler_sleep_ns(us * 1000); };
const auto send_ipi =
[&](uint8_t processor, uint32_t data, uint64_t ud)
@ -367,6 +388,31 @@ namespace Kernel
void APIC::enable()
{
write_to_local_apic(LAPIC_SIV_REG, read_from_local_apic(LAPIC_SIV_REG) | 0x1FF);
initialize_timer();
}
static SpinLock s_timer_init_lock;
void APIC::initialize_timer()
{
{
constexpr uint64_t measuring_duration_ms = 100;
SpinLockGuard _(s_timer_init_lock);
write_to_local_apic(LAPIC_TIMER_LVT, TimerModeOneShot | TimerMask);
write_to_local_apic(LAPIC_TIMER_DIVIDE_REG, DivideBy2);
write_to_local_apic(LAPIC_TIMER_INITIAL_REG, 0xFFFFFFFF);
SystemTimer::get().pre_scheduler_sleep_ns(measuring_duration_ms * 1'000'000);
const uint32_t counter = read_from_local_apic(LAPIC_TIMER_CURRENT_REG);
m_lapic_timer_frequency_hz = static_cast<uint64_t>(0xFFFFFFFF - counter) * 2 * (1000 / measuring_duration_ms);
dprintln("CPU {}: lapic timer frequency: {} Hz", Kernel::Processor::current_id(), m_lapic_timer_frequency_hz);
}
write_to_local_apic(LAPIC_TIMER_LVT, TimerModePeriodic | (IRQ_VECTOR_BASE + IRQ_TIMER));
write_to_local_apic(LAPIC_TIMER_INITIAL_REG, m_lapic_timer_frequency_hz / 2 / 100);
}
uint32_t APIC::read_from_local_apic(ptrdiff_t offset)

9
kernel/kernel/IDT.cpp
View File

@ -333,6 +333,13 @@ done:
Processor::handle_ipi();
}
extern "C" void cpp_timer_handler()
{
ASSERT(InterruptController::get().is_in_service(IRQ_TIMER));
InterruptController::get().eoi(IRQ_TIMER);
Processor::scheduler().timer_interrupt();
}
extern "C" void cpp_irq_handler(uint32_t irq)
{
if (g_paniced)
@ -409,6 +416,7 @@ done:
extern "C" void asm_yield_handler();
extern "C" void asm_ipi_handler();
extern "C" void asm_timer_handler();
extern "C" void asm_syscall_handler();
IDT* IDT::create()
@ -428,6 +436,7 @@ done:
idt->register_interrupt_handler(IRQ_VECTOR_BASE + IRQ_YIELD, asm_yield_handler);
idt->register_interrupt_handler(IRQ_VECTOR_BASE + IRQ_IPI, asm_ipi_handler);
idt->register_interrupt_handler(IRQ_VECTOR_BASE + IRQ_TIMER, asm_timer_handler);
idt->register_syscall_handler(0x80, asm_syscall_handler);

14
kernel/kernel/Processor.cpp
View File

@ -20,7 +20,7 @@ namespace Kernel
static BAN::Atomic<uint8_t> s_processors_created { 0 };
// 32 bit milli seconds are definitely enough as APs start on boot
static BAN::Atomic<uint32_t> s_first_ap_ready_ms { 0 };
static BAN::Atomic<uint32_t> s_first_ap_ready_ms { static_cast<uint32_t>(-1) };
static BAN::Array<Processor, 0xFF> s_processors;
static BAN::Array<ProcessorID, 0xFF> s_processor_ids { PROCESSOR_NONE };
@ -130,7 +130,7 @@ namespace Kernel
// wait until first AP is ready
const uint64_t timeout_ms = SystemTimer::get().ms_since_boot() + 1000;
while (s_first_ap_ready_ms == 0)
while (s_first_ap_ready_ms == static_cast<uint32_t>(-1))
{
if (SystemTimer::get().ms_since_boot() >= timeout_ms)
{
@ -150,7 +150,7 @@ namespace Kernel
ASSERT(s_processor_ids[lookup_index] == PROCESSOR_NONE);
s_processor_ids[lookup_index] = current_id();
uint32_t expected = 0;
uint32_t expected = static_cast<uint32_t>(-1);
s_first_ap_ready_ms.compare_exchange(expected, SystemTimer::get().ms_since_boot());
}
@ -209,8 +209,6 @@ namespace Kernel
}
);
bool should_preempt = false;
if (pending)
{
// reverse smp message queue from LIFO to FIFO
@ -245,9 +243,6 @@ namespace Kernel
case SMPMessage::Type::UnblockThread:
processor.m_scheduler->handle_unblock_request(message->unblock_thread);
break;
case SMPMessage::Type::SchedulerPreemption:
should_preempt = true;
break;
}
last_handled = message;
@ -262,9 +257,6 @@ namespace Kernel
);
}
if (should_preempt)
processor.m_scheduler->preempt();
set_interrupt_state(state);
}

19
kernel/kernel/Scheduler.cpp
View File

@ -294,7 +294,7 @@ namespace Kernel
Processor::yield();
}
void Scheduler::preempt()
void Scheduler::timer_interrupt()
{
ASSERT(Processor::get_interrupt_state() == InterruptState::Disabled);
@ -321,23 +321,6 @@ namespace Kernel
}
}
void Scheduler::timer_interrupt()
{
ASSERT(Processor::get_interrupt_state() == InterruptState::Disabled);
// FIXME: all processors should LAPIC for their preemption
if (Processor::is_smp_enabled())
{
ASSERT(Processor::current_is_bsb());
Processor::broadcast_smp_message({
.type = Processor::SMPMessage::Type::SchedulerPreemption,
.scheduler_preemption = 0 // dummy value
});
}
preempt();
}
void Scheduler::handle_unblock_request(const UnblockRequest& request)
{
ASSERT(Processor::get_interrupt_state() == InterruptState::Disabled);

36
kernel/kernel/Timer/HPET.cpp
View File

@ -254,9 +254,9 @@ namespace Kernel
void HPET::handle_irq()
{
auto& regs = registers();
{
auto& regs = registers();
SpinLockGuard _(m_lock);
uint64_t current_ticks;
@ -272,7 +272,8 @@ namespace Kernel
m_last_ticks = current_ticks;
}
Processor::scheduler().timer_interrupt();
if (should_invoke_scheduler())
Processor::scheduler().timer_interrupt();
}
uint64_t HPET::ms_since_boot() const
@ -303,4 +304,33 @@ namespace Kernel
};
}
void HPET::pre_scheduler_sleep_ns(uint64_t ns)
{
auto& regs = registers();
const uint64_t target_ticks = BAN::Math::div_round_up<uint64_t>(ns * FS_PER_NS, regs.counter_clk_period);
if (m_is_64bit)
{
const uint64_t target_counter = regs.main_counter.full + target_ticks;
while (regs.main_counter.full < target_counter)
__builtin_ia32_pause();
}
else
{
uint64_t elapsed_ticks = 0;
uint64_t last_counter = regs.main_counter.low;
while (elapsed_ticks < target_ticks)
{
const uint64_t current_counter = regs.main_counter.low;
if (last_counter <= current_counter)
elapsed_ticks += current_counter - last_counter;
else
elapsed_ticks += 0xFFFFFFFF + current_counter - last_counter;
last_counter = current_counter;
__builtin_ia32_pause();
}
}
}
}

76
kernel/kernel/Timer/PIT.cpp
View File

@ -18,17 +18,19 @@
#define ACCESS_HI 0x10
#define ACCESS_LO 0x20
#define MODE_SQUARE_WAVE 0x06
#define MODE_RATE_GENERATOR 0x05
#define BASE_FREQUENCY 1193182
#define TICKS_PER_SECOND 1000
#define MS_PER_S 1'000
#define NS_PER_MS 1'000'000
#define NS_PER_S 1'000'000'000
namespace Kernel
{
constexpr uint16_t s_ticks_per_ms = BASE_FREQUENCY / 1000;
BAN::ErrorOr<BAN::UniqPtr<PIT>> PIT::create()
{
PIT* pit = new PIT();
@ -40,12 +42,9 @@ namespace Kernel
void PIT::initialize()
{
constexpr uint16_t timer_reload = BASE_FREQUENCY / TICKS_PER_SECOND;
IO::outb(PIT_CTL, SELECT_CHANNEL0 | ACCESS_LO | ACCESS_HI | MODE_SQUARE_WAVE);
IO::outb(TIMER0_CTL, (timer_reload >> 0) & 0xff);
IO::outb(TIMER0_CTL, (timer_reload >> 8) & 0xff);
IO::outb(PIT_CTL, SELECT_CHANNEL0 | ACCESS_LO | ACCESS_HI | MODE_RATE_GENERATOR);
IO::outb(TIMER0_CTL, (s_ticks_per_ms >> 0) & 0xff);
IO::outb(TIMER0_CTL, (s_ticks_per_ms >> 8) & 0xff);
MUST(InterruptController::get().reserve_irq(PIT_IRQ));
set_irq(PIT_IRQ);
@ -56,34 +55,77 @@ namespace Kernel
{
{
SpinLockGuard _(m_lock);
m_system_time++;
m_system_time_ms++;
}
Processor::scheduler().timer_interrupt();
if (should_invoke_scheduler())
Processor::scheduler().timer_interrupt();
}
uint64_t PIT::read_counter() const
uint64_t PIT::read_counter_ns() const
{
SpinLockGuard _(m_lock);
return m_system_time;
// send latch command
IO::outb(PIT_CTL, SELECT_CHANNEL0);
// read ticks
uint64_t ticks_this_ms { 0 };
ticks_this_ms |= IO::inb(TIMER0_CTL);
ticks_this_ms |= IO::inb(TIMER0_CTL) << 8;
ticks_this_ms = s_ticks_per_ms - ticks_this_ms;
const uint64_t ns_this_ms = ticks_this_ms * NS_PER_S / BASE_FREQUENCY;
return (m_system_time_ms * NS_PER_MS) + ns_this_ms;
}
uint64_t PIT::ms_since_boot() const
{
return read_counter() * (MS_PER_S / TICKS_PER_SECOND);
return read_counter_ns() / NS_PER_MS;
}
uint64_t PIT::ns_since_boot() const
{
return read_counter() * (NS_PER_S / TICKS_PER_SECOND);
return read_counter_ns();
}
timespec PIT::time_since_boot() const
{
uint64_t ticks = read_counter();
uint64_t ns = read_counter_ns();
return timespec {
.tv_sec = ticks / TICKS_PER_SECOND,
.tv_nsec = (long)((ticks % TICKS_PER_SECOND) * (NS_PER_S / TICKS_PER_SECOND))
.tv_sec = static_cast<time_t>(ns / NS_PER_S),
.tv_nsec = static_cast<long>(ns % NS_PER_S)
};
}
void PIT::pre_scheduler_sleep_ns(uint64_t ns)
{
const uint64_t target_ticks = BAN::Math::div_round_up<uint64_t>(ns * BASE_FREQUENCY, NS_PER_S);
SpinLockGuard _(m_lock);
IO::outb(PIT_CTL, SELECT_CHANNEL0 | ACCESS_LO | ACCESS_HI | MODE_RATE_GENERATOR);
IO::outb(TIMER0_CTL, 0);
IO::outb(TIMER0_CTL, 0);
IO::outb(PIT_CTL, SELECT_CHANNEL0 | ACCESS_LO | MODE_RATE_GENERATOR);
IO::outb(TIMER0_CTL, 0xFF);
uint64_t elapsed_ticks = 0;
uint8_t last_ticks = IO::inb(TIMER0_CTL);
while (elapsed_ticks < target_ticks)
{
const uint8_t current_ticks = IO::inb(TIMER0_CTL);
if (last_ticks <= current_ticks)
elapsed_ticks += current_ticks - last_ticks;
else
elapsed_ticks += 0xFF + current_ticks - last_ticks;
last_ticks = current_ticks;
}
IO::outb(PIT_CTL, SELECT_CHANNEL0 | ACCESS_LO | ACCESS_HI | MODE_RATE_GENERATOR);
IO::outb(TIMER0_CTL, (s_ticks_per_ms >> 0) & 0xFF);
IO::outb(TIMER0_CTL, (s_ticks_per_ms >> 8) & 0xFF);
}
}

9
kernel/kernel/Timer/Timer.cpp
View File

@ -69,6 +69,11 @@ namespace Kernel
return m_timer->time_since_boot();
}
void SystemTimer::pre_scheduler_sleep_ns(uint64_t ns)
{
return m_timer->pre_scheduler_sleep_ns(ns);
}
void SystemTimer::sleep_ns(uint64_t ns) const
{
if (ns == 0)
@ -76,10 +81,6 @@ namespace Kernel
const uint64_t wake_time_ns = ns_since_boot() + ns;
Processor::scheduler().block_current_thread(nullptr, wake_time_ns);
//const uint64_t current_time_ns = ns_since_boot();
//if (current_time_ns < wake_time_ns)
// dwarnln("sleep woke {} ms too soon", BAN::Math::div_round_up<uint64_t>(wake_time_ns - current_time_ns, 1'000'000));
}
timespec SystemTimer::real_time() const

7
kernel/kernel/kernel.cpp
View File

@ -1,4 +1,5 @@
#include <kernel/ACPI/ACPI.h>
#include <kernel/APIC.h>
#include <kernel/Arch.h>
#include <kernel/BootInfo.h>
#include <kernel/Debug.h>
@ -167,6 +168,12 @@ extern "C" void kernel_main(uint32_t boot_magic, uint32_t boot_info)
Random::initialize();
dprintln("RNG initialized");
if (InterruptController::get().is_using_apic())
{
SystemTimer::get().dont_invoke_scheduler();
static_cast<APIC&>(InterruptController::get()).initialize_timer();
}
Processor::wait_until_processors_ready();
MUST(Processor::scheduler().initialize());