Kernel: ATA now uses irqs instead of polling

Reading is now much slower at ~500 kB/s it was around 3 MB/s.
This is probably mostly due semaphore blocking taking atleast
until next reschedule (1 ms itervals). This will be a problem
as long as we are using only single processor.

I could try to use {READ/WRITE}_MULTIPLE commands, but since
most of the disk reads are 2 sectors (inode block size) this
will at most double the speed.

Most efficient speed up would of course be caching disk access
data and inodes overall.
This commit is contained in:
Bananymous
2023-04-02 23:56:01 +03:00
parent dcc174b62e
commit 46d65471d9
10 changed files with 449 additions and 296 deletions

View File

@@ -42,8 +42,13 @@ namespace Kernel
for (auto* device : controller->devices())
{
s_instance->add_device(device);
for (auto* partition : device->partitions())
s_instance->add_device(partition);
if (auto res = device->initialize_partitions(); res.is_error())
dprintln("{}: {}", device->name(), res.error());
else
{
for (auto* partition : device->partitions())
s_instance->add_device(partition);
}
}
}
break;

View File

@@ -0,0 +1,318 @@
#include <BAN/ScopeGuard.h>
#include <kernel/IDT.h>
#include <kernel/InterruptController.h>
#include <kernel/IO.h>
#include <kernel/LockGuard.h>
#include <kernel/Storage/ATADevice.h>
#include <kernel/Storage/ATABus.h>
#include <kernel/Storage/ATADefinitions.h>
#include <kernel/CriticalScope.h>
namespace Kernel
{
static void bus_irq_handler0();
static void bus_irq_handler1();
struct BusIRQ
{
ATABus* bus { nullptr };
void (*handler)() { nullptr };
uint8_t irq { 0 };
};
static BusIRQ s_bus_irqs[] {
{ nullptr, bus_irq_handler0, 0 },
{ nullptr, bus_irq_handler1, 0 },
};
static void bus_irq_handler0() { ASSERT(s_bus_irqs[0].bus); s_bus_irqs[0].bus->on_irq(); }
static void bus_irq_handler1() { ASSERT(s_bus_irqs[1].bus); s_bus_irqs[1].bus->on_irq(); }
static void register_bus_irq_handler(ATABus* bus, uint8_t irq)
{
for (uint8_t i = 0; i < sizeof(s_bus_irqs) / sizeof(BusIRQ); i++)
{
if (s_bus_irqs[i].bus == nullptr)
{
s_bus_irqs[i].bus = bus;
s_bus_irqs[i].irq = irq;
IDT::register_irq_handler(irq, s_bus_irqs[i].handler);
InterruptController::get().enable_irq(irq);
return;
}
}
ASSERT_NOT_REACHED();
}
ATABus* ATABus::create(ATAController* controller, uint16_t base, uint16_t ctrl, uint8_t irq)
{
ATABus* bus = new ATABus(controller, base, ctrl);
ASSERT(bus);
bus->initialize(irq);
return bus;
}
void ATABus::initialize(uint8_t irq)
{
register_bus_irq_handler(this, irq);
for (uint8_t i = 0; i < 2; i++)
{
m_devices[i] = new ATADevice(this);
ATADevice* device = m_devices[i];
ASSERT(device);
BAN::ScopeGuard guard([this, i] { m_devices[i]->unref(); m_devices[i] = nullptr; });
uint16_t identify_buffer[256];
auto type = identify(device, identify_buffer);
if (type == DeviceType::None)
continue;
auto res = device->initialize(type, identify_buffer);
if (res.is_error())
{
dprintln("{}", res.error());
continue;
}
guard.disable();
}
// Enable disk interrupts
for (int i = 0; i < 2; i++)
{
if (!m_devices[i])
continue;
select_device(m_devices[i]);
io_write(ATA_PORT_CONTROL, 0);
}
}
void ATABus::select_device(const ATADevice* device)
{
uint8_t device_index = this->device_index(device);
io_write(ATA_PORT_DRIVE_SELECT, 0xA0 | (device_index << 4));
PIT::sleep(1);
}
ATABus::DeviceType ATABus::identify(const ATADevice* device, uint16_t* buffer)
{
select_device(device);
// Disable interrupts
io_write(ATA_PORT_CONTROL, ATA_CONTROL_nIEN);
io_write(ATA_PORT_COMMAND, ATA_COMMAND_IDENTIFY);
PIT::sleep(1);
// No device on port
if (io_read(ATA_PORT_STATUS) == 0)
return DeviceType::None;
DeviceType type = DeviceType::ATA;
if (wait(true).is_error())
{
uint8_t lba1 = io_read(ATA_PORT_LBA1);
uint8_t lba2 = io_read(ATA_PORT_LBA2);
if (lba1 == 0x14 && lba2 == 0xEB)
type = DeviceType::ATAPI;
else if (lba1 == 0x69 && lba2 == 0x96)
type = DeviceType::ATAPI;
else
{
dprintln("Unsupported device type");
return DeviceType::None;
}
io_write(ATA_PORT_COMMAND, ATA_COMMAND_IDENTIFY_PACKET);
PIT::sleep(1);
if (auto res = wait(true); res.is_error())
{
dprintln("Fatal error: {}", res.error());
return DeviceType::None;
}
}
read_buffer(ATA_PORT_DATA, buffer, 256);
return type;
}
void ATABus::on_irq()
{
ASSERT(!m_has_got_irq);
if (io_read(ATA_PORT_STATUS) & ATA_STATUS_ERR)
dprintln("ATA Error: {}", error());
m_has_got_irq = true;
m_semaphore.unblock();
}
void ATABus::block_until_irq()
{
if (!m_has_got_irq)
m_semaphore.block();
m_has_got_irq = false;
}
uint8_t ATABus::io_read(uint16_t port)
{
if (port <= 0x07)
return IO::inb(m_base + port);
if (0x10 <= port && port <= 0x11)
return IO::inb(m_ctrl + port - 0x10);
ASSERT_NOT_REACHED();
}
void ATABus::read_buffer(uint16_t port, uint16_t* buffer, size_t words)
{
if (port <= 0x07)
return IO::insw(m_base + port - 0x00, buffer, words);
if (0x10 <= port && port <= 0x11)
return IO::insw(m_ctrl + port - 0x10, buffer, words);
ASSERT_NOT_REACHED();
}
void ATABus::io_write(uint16_t port, uint8_t data)
{
if (port <= 0x07)
return IO::outb(m_base + port, data);
if (0x10 <= port && port <= 0x11)
return IO::outb(m_ctrl + port - 0x10, data);
ASSERT_NOT_REACHED();
}
void ATABus::write_buffer(uint16_t port, const uint16_t* buffer, size_t words)
{
uint16_t io_port = 0;
if (port <= 0x07)
io_port = m_base + port;
if (0x10 <= port && port <= 0x11)
io_port = m_ctrl + port - 0x10;
ASSERT(io_port);
for (size_t i = 0; i < words; i++)
IO::outw(io_port, buffer[i]);
}
BAN::ErrorOr<void> ATABus::wait(bool wait_drq)
{
for (uint32_t i = 0; i < 4; i++)
io_read(ATA_PORT_ALT_STATUS);
uint8_t status = ATA_STATUS_BSY;
while (status & ATA_STATUS_BSY)
status = io_read(ATA_PORT_STATUS);
while (wait_drq && !(status & ATA_STATUS_DRQ))
{
if (status & ATA_STATUS_ERR)
return error();
if (status & ATA_STATUS_DF)
return BAN::Error::from_errno(EIO);
status = io_read(ATA_PORT_STATUS);
}
return {};
}
BAN::Error ATABus::error()
{
uint8_t err = io_read(ATA_PORT_ERROR);
if (err & ATA_ERROR_AMNF)
return BAN::Error::from_c_string("Address mark not found.");
if (err & ATA_ERROR_TKZNF)
return BAN::Error::from_c_string("Track zero not found.");
if (err & ATA_ERROR_ABRT)
return BAN::Error::from_c_string("Aborted command.");
if (err & ATA_ERROR_MCR)
return BAN::Error::from_c_string("Media change request.");
if (err & ATA_ERROR_IDNF)
return BAN::Error::from_c_string("ID not found.");
if (err & ATA_ERROR_MC)
return BAN::Error::from_c_string("Media changed.");
if (err & ATA_ERROR_UNC)
return BAN::Error::from_c_string("Uncorrectable data error.");
if (err & ATA_ERROR_BBK)
return BAN::Error::from_c_string("Bad Block detected.");
return BAN::Error::from_c_string("No error");
}
uint8_t ATABus::device_index(const ATADevice* device) const
{
ASSERT(device == m_devices[0] || device == m_devices[1]);
return (device == m_devices[0]) ? 0 : 1;
}
BAN::ErrorOr<void> ATABus::read(ATADevice* device, uint64_t lba, uint8_t sector_count, uint8_t* buffer)
{
if (lba + sector_count > device->m_lba_count)
return BAN::Error::from_c_string("Attempted to read outside of the device boundaries");
LockGuard _(m_lock);
if (lba < (1 << 28))
{
// LBA28
io_write(ATA_PORT_DRIVE_SELECT, 0xE0 | (device_index(device) << 4) | ((lba >> 24) & 0x0F));
io_write(ATA_PORT_SECTOR_COUNT, sector_count);
io_write(ATA_PORT_LBA0, (uint8_t)(lba >> 0));
io_write(ATA_PORT_LBA1, (uint8_t)(lba >> 8));
io_write(ATA_PORT_LBA2, (uint8_t)(lba >> 16));
io_write(ATA_PORT_COMMAND, ATA_COMMAND_READ_SECTORS);
for (uint32_t sector = 0; sector < sector_count; sector++)
{
block_until_irq();
read_buffer(ATA_PORT_DATA, (uint16_t*)buffer + sector * device->m_sector_words, device->m_sector_words);
}
}
else
{
// LBA48
ASSERT(false);
}
return {};
}
BAN::ErrorOr<void> ATABus::write(ATADevice* device, uint64_t lba, uint8_t sector_count, const uint8_t* buffer)
{
if (lba + sector_count > device->m_lba_count)
return BAN::Error::from_c_string("Attempted to write outside of the device boundaries");
LockGuard _(m_lock);
if (lba < (1 << 28))
{
// LBA28
io_write(ATA_PORT_DRIVE_SELECT, 0xE0 | (device_index(device) << 4) | ((lba >> 24) & 0x0F));
io_write(ATA_PORT_SECTOR_COUNT, sector_count);
io_write(ATA_PORT_LBA0, (uint8_t)(lba >> 0));
io_write(ATA_PORT_LBA1, (uint8_t)(lba >> 8));
io_write(ATA_PORT_LBA2, (uint8_t)(lba >> 16));
io_write(ATA_PORT_COMMAND, ATA_COMMAND_WRITE_SECTORS);
for (uint32_t sector = 0; sector < sector_count; sector++)
{
write_buffer(ATA_PORT_DATA, (uint16_t*)buffer + sector * device->m_sector_words, device->m_sector_words);
block_until_irq();
}
}
else
{
// LBA48
ASSERT(false);
}
io_write(ATA_PORT_COMMAND, ATA_COMMAND_CACHE_FLUSH);
block_until_irq();
return {};
}
}

View File

@@ -1,5 +1,6 @@
#include <BAN/ScopeGuard.h>
#include <kernel/LockGuard.h>
#include <kernel/Storage/ATABus.h>
#include <kernel/Storage/ATAController.h>
#include <kernel/Storage/ATADefinitions.h>
#include <kernel/Storage/ATADevice.h>
@@ -45,94 +46,30 @@ namespace Kernel
buses[1].ctrl = pci_device.read_dword(0x1C) & 0xFFFFFFFC;
}
for (uint8_t drive = 0; drive < 4; drive++)
{
auto device = ATADevice::create(this, buses[drive / 2].base, buses[drive / 2].ctrl, drive);
if (device.is_error())
{
if (strlen(device.error().get_message()) > 0)
dprintln("{}", device.error());
continue;
}
add_device(device.value());
}
for (StorageDevice* device_ : devices())
{
ATADevice& device = *(ATADevice*)device_;
dprintln("Initialized drive {} ({} MiB) model {}", device.name(), device.total_size() / (1024 * 1024), device.model());
}
m_buses[0] = ATABus::create(this, buses[0].base, buses[0].ctrl, 14);
m_buses[1] = ATABus::create(this, buses[1].base, buses[1].ctrl, 15);
return {};
}
BAN::ErrorOr<void> ATAController::read(ATADevice* device, uint64_t lba, uint8_t sector_count, uint8_t* buffer)
uint8_t ATAController::next_device_index() const
{
if (lba + sector_count > device->m_lba_count)
return BAN::Error::from_c_string("Attempted to read outside of the device boundaries");
LockGuard _(m_lock);
if (lba < (1 << 28))
{
// LBA28
TRY(device->wait(false));
device->io_write(ATA_PORT_DRIVE_SELECT, 0xE0 | device->m_slave_bit | ((lba >> 24) & 0x0F));
device->io_write(ATA_PORT_SECTOR_COUNT, sector_count);
device->io_write(ATA_PORT_LBA0, (uint8_t)(lba >> 0));
device->io_write(ATA_PORT_LBA1, (uint8_t)(lba >> 8));
device->io_write(ATA_PORT_LBA2, (uint8_t)(lba >> 16));
device->io_write(ATA_PORT_COMMAND, ATA_COMMAND_READ_SECTORS);
for (uint32_t sector = 0; sector < sector_count; sector++)
{
TRY(device->wait(true));
device->read_buffer(ATA_PORT_DATA, (uint16_t*)buffer + sector * device->m_sector_words, device->m_sector_words);
}
}
else
{
// LBA48
ASSERT(false);
}
return {};
static uint8_t index = 0;
return index++;
}
BAN::ErrorOr<void> ATAController::write(ATADevice* device, uint64_t lba, uint8_t sector_count, const uint8_t* buffer)
BAN::Vector<StorageDevice*> ATAController::devices()
{
if (lba + sector_count > device->m_lba_count)
return BAN::Error::from_c_string("Attempted to write outside of the device boundaries");
LockGuard _(m_lock);
if (lba < (1 << 28))
{
// LBA28
TRY(device->wait(false));
device->io_write(ATA_PORT_DRIVE_SELECT, 0xE0 | device->m_slave_bit | ((lba >> 24) & 0x0F));
device->io_write(ATA_PORT_SECTOR_COUNT, sector_count);
device->io_write(ATA_PORT_LBA0, (uint8_t)(lba >> 0));
device->io_write(ATA_PORT_LBA1, (uint8_t)(lba >> 8));
device->io_write(ATA_PORT_LBA2, (uint8_t)(lba >> 16));
device->io_write(ATA_PORT_COMMAND, ATA_COMMAND_WRITE_SECTORS);
for (uint32_t sector = 0; sector < sector_count; sector++)
{
TRY(device->wait(false));
device->write_buffer(ATA_PORT_DATA, (uint16_t*)buffer + sector * device->m_sector_words, device->m_sector_words);
}
}
else
{
// LBA48
ASSERT(false);
}
TRY(device->wait(false));
device->io_write(ATA_PORT_COMMAND, ATA_COMMAND_CACHE_FLUSH);
return {};
BAN::Vector<StorageDevice*> devices;
if (m_buses[0]->m_devices[0])
MUST(devices.push_back(m_buses[0]->m_devices[0]));
if (m_buses[0]->m_devices[1])
MUST(devices.push_back(m_buses[0]->m_devices[1]));
if (m_buses[1]->m_devices[0])
MUST(devices.push_back(m_buses[1]->m_devices[0]));
if (m_buses[1]->m_devices[1])
MUST(devices.push_back(m_buses[1]->m_devices[1]));
return devices;
}
}

View File

@@ -1,81 +1,30 @@
#include <BAN/ScopeGuard.h>
#include <kernel/IO.h>
#include <kernel/Storage/ATAController.h>
#include <kernel/Storage/ATABus.h>
#include <kernel/Storage/ATADefinitions.h>
#include <kernel/Storage/ATADevice.h>
namespace Kernel
{
BAN::ErrorOr<ATADevice*> ATADevice::create(ATAController* controller, uint16_t base, uint16_t ctrl, uint8_t index)
BAN::ErrorOr<void> ATADevice::initialize(ATABus::DeviceType type, const uint16_t* identify_buffer)
{
ATADevice* device = new ATADevice(controller, base, ctrl, index);
if (device == nullptr)
return BAN::Error::from_errno(ENOMEM);
BAN::ScopeGuard guard([device] { device->unref(); });
TRY(device->initialize());
guard.disable();
return device;
}
m_type = type;
BAN::ErrorOr<void> ATADevice::initialize()
{
io_write(ATA_PORT_CONTROL, ATA_CONTROL_nIEN);
m_signature = identify_buffer[ATA_IDENTIFY_SIGNATURE];
m_capabilities = identify_buffer[ATA_IDENTIFY_CAPABILITIES];
io_write(ATA_PORT_DRIVE_SELECT, 0xA0 | m_slave_bit);
PIT::sleep(1);
io_write(ATA_PORT_COMMAND, ATA_COMMAND_IDENTIFY);
PIT::sleep(1);
if (io_read(ATA_PORT_STATUS) == 0)
return BAN::Error::from_c_string("");
uint8_t status = 0;
while (true)
{
status = io_read(ATA_PORT_STATUS);
if (status & ATA_STATUS_ERR)
break;
if (!(status & ATA_STATUS_BSY) && (status && ATA_STATUS_DRQ))
break;
}
if (status & ATA_STATUS_ERR)
{
uint8_t lba1 = io_read(ATA_PORT_LBA1);
uint8_t lba2 = io_read(ATA_PORT_LBA2);
if (lba1 == 0x14 && lba2 == 0xEB)
m_type = ATADevice::DeviceType::ATAPI;
else if (lba1 == 0x69 && lba2 == 0x96)
m_type = ATADevice::DeviceType::ATAPI;
else
return BAN::Error::from_c_string("Not ATA/ATAPI device");
io_write(ATA_PORT_COMMAND, ATA_COMMAND_IDENTIFY_PACKET);
PIT::sleep(1);
}
else
{
m_type = ATADevice::DeviceType::ATA;
}
uint16_t buffer[256] {};
read_buffer(ATA_PORT_DATA, buffer, 256);
m_signature = *(uint16_t*)(buffer + ATA_IDENTIFY_SIGNATURE);
m_capabilities = *(uint16_t*)(buffer + ATA_IDENTIFY_CAPABILITIES);
m_command_set = *(uint32_t*)(buffer + ATA_IDENTIFY_COMMAND_SET);
m_command_set = 0;
m_command_set |= (uint32_t)(identify_buffer[ATA_IDENTIFY_COMMAND_SET + 0] << 0);
m_command_set |= (uint32_t)(identify_buffer[ATA_IDENTIFY_COMMAND_SET + 1] << 16);
if (!(m_capabilities & ATA_CAPABILITIES_LBA))
return BAN::Error::from_c_string("Device does not support LBA addressing");
if ((buffer[ATA_IDENTIFY_SECTOR_INFO] & (1 << 15)) == 0 &&
(buffer[ATA_IDENTIFY_SECTOR_INFO] & (1 << 14)) != 0 &&
(buffer[ATA_IDENTIFY_SECTOR_INFO] & (1 << 12)) != 0)
if ((identify_buffer[ATA_IDENTIFY_SECTOR_INFO] & (1 << 15)) == 0 &&
(identify_buffer[ATA_IDENTIFY_SECTOR_INFO] & (1 << 14)) != 0 &&
(identify_buffer[ATA_IDENTIFY_SECTOR_INFO] & (1 << 12)) != 0)
{
m_sector_words = *(uint32_t*)(buffer + ATA_IDENTIFY_SECTOR_WORDS);
m_sector_words = *(uint32_t*)(identify_buffer + ATA_IDENTIFY_SECTOR_WORDS);
}
else
{
@@ -84,127 +33,41 @@ namespace Kernel
m_lba_count = 0;
if (m_command_set & ATA_COMMANDSET_LBA48_SUPPORTED)
m_lba_count = *(uint64_t*)(buffer + ATA_IDENTIFY_LBA_COUNT_EXT);
m_lba_count = *(uint64_t*)(identify_buffer + ATA_IDENTIFY_LBA_COUNT_EXT);
if (m_lba_count < (1 << 28))
m_lba_count = *(uint32_t*)(buffer + ATA_IDENTIFY_LBA_COUNT);
m_lba_count = *(uint32_t*)(identify_buffer + ATA_IDENTIFY_LBA_COUNT);
for (int i = 0; i < 20; i++)
{
uint16_t word = buffer[ATA_IDENTIFY_MODEL + i];
uint16_t word = identify_buffer[ATA_IDENTIFY_MODEL + i];
m_model[2 * i + 0] = word >> 8;
m_model[2 * i + 1] = word & 0xFF;
}
m_model[40] = 0;
m_index = m_bus->controller()->next_device_index();
m_device_name[0] = 'h';
m_device_name[1] = 'd';
m_device_name[2] = 'a' + m_index;
m_device_name[3] = '\0';
TRY(initialize_partitions());
dprintln("{} {} MB", m_device_name, total_size() / 1024 / 1024);
return {};
}
BAN::ErrorOr<void> ATADevice::read_sectors(uint64_t lba, uint8_t sector_count, uint8_t* buffer)
{
TRY(m_controller->read(this, lba, sector_count, buffer));
TRY(m_bus->read(this, lba, sector_count, buffer));
return {};
}
BAN::ErrorOr<void> ATADevice::write_sectors(uint64_t lba, uint8_t sector_count, const uint8_t* buffer)
{
TRY(m_controller->write(this, lba, sector_count, buffer));
TRY(m_bus->write(this, lba, sector_count, buffer));
return {};
}
uint8_t ATADevice::io_read(uint16_t port)
{
if (port <= 0x07)
return IO::inb(m_base + port);
if (0x10 <= port && port <= 0x11)
return IO::inb(m_ctrl + port - 0x10);
ASSERT_NOT_REACHED();
}
void ATADevice::read_buffer(uint16_t port, uint16_t* buffer, size_t words)
{
if (port <= 0x07)
return IO::insw(m_base + port - 0x00, buffer, words);
if (0x10 <= port && port <= 0x11)
return IO::insw(m_ctrl + port - 0x10, buffer, words);
ASSERT_NOT_REACHED();
}
void ATADevice::io_write(uint16_t port, uint8_t data)
{
if (port <= 0x07)
return IO::outb(m_base + port, data);
if (0x10 <= port && port <= 0x11)
return IO::outb(m_ctrl + port - 0x10, data);
ASSERT_NOT_REACHED();
}
void ATADevice::write_buffer(uint16_t port, const uint16_t* buffer, size_t words)
{
uint16_t io_port = 0;
if (port <= 0x07)
io_port = m_base + port;
if (0x10 <= port && port <= 0x11)
io_port = m_ctrl + port - 0x10;
ASSERT(io_port);
for (size_t i = 0; i < words; i++)
IO::outw(io_port, buffer[i]);
}
BAN::ErrorOr<void> ATADevice::wait(bool wait_drq)
{
for (uint32_t i = 0; i < 4; i++)
io_read(ATA_PORT_ALT_STATUS);
uint8_t status = ATA_STATUS_BSY;
while (status & ATA_STATUS_BSY)
status = io_read(ATA_PORT_STATUS);
while (wait_drq && !(status & ATA_STATUS_DRQ))
{
if (status & ATA_STATUS_ERR)
return error();
if (status & ATA_STATUS_DF)
return BAN::Error::from_errno(EIO);
status = io_read(ATA_PORT_STATUS);
}
return {};
}
BAN::Error ATADevice::error()
{
uint8_t err = io_read(ATA_PORT_ERROR);
if (err & ATA_ERROR_AMNF)
return BAN::Error::from_c_string("Address mark not found.");
if (err & ATA_ERROR_TKZNF)
return BAN::Error::from_c_string("Track zero not found.");
if (err & ATA_ERROR_ABRT)
return BAN::Error::from_c_string("Aborted command.");
if (err & ATA_ERROR_MCR)
return BAN::Error::from_c_string("Media change request.");
if (err & ATA_ERROR_IDNF)
return BAN::Error::from_c_string("ID not found.");
if (err & ATA_ERROR_MC)
return BAN::Error::from_c_string("Media changed.");
if (err & ATA_ERROR_UNC)
return BAN::Error::from_c_string("Uncorrectable data error.");
if (err & ATA_ERROR_BBK)
return BAN::Error::from_c_string("Bad Block detected.");
ASSERT_NOT_REACHED();
}
dev_t ATADevice::dev() const
{
ASSERT(m_controller);
return m_controller->dev();
}
BAN::ErrorOr<size_t> ATADevice::read(size_t offset, void* buffer, size_t bytes)
{
if (offset % sector_size() || bytes % sector_size())