Kernel: Implement AHCI driver

SATA drives can now be used with banan-os. This allows much faster
disk access (writing 10 MiB from 30s to 1.5s). This can definitely
be optimized but the main slow down is probably the whole disk
structure in the os.

AHCI drive is now the default when running qemu.

BAN/include/BAN/Optional.h

@@ -13,6 +13,8 @@ namespace BAN
 	{
 	public:
 		Optional();
+		Optional(Optional&&);
+		Optional(const Optional&);
 		Optional(const T&);
 		Optional(T&&);
 		template<typename... Args>
@@ -20,8 +22,8 @@ namespace BAN
 
 		~Optional();
 
-		Optional& operator=(const Optional&);
 		Optional& operator=(Optional&&);
+		Optional& operator=(const Optional&);
 
 		template<typename... Args>
 		Optional& emplace(Args&&...);
@@ -34,9 +36,9 @@ namespace BAN
 
 		bool has_value() const;
 
-		T&& release_value();
-		const T& value() const;
+		T release_value();
 		T& value();
+		const T& value() const;
 
 		void clear();
 
@@ -50,6 +52,22 @@ namespace BAN
 		: m_has_value(false)
 	{}
 
+	template<typename T>
+	Optional<T>::Optional(Optional<T>&& other)
+		: m_has_value(other.has_value())
+	{
+		if (other.has_value())
+			new (m_storage) T(move(other.release_value()));
+	}
+
+	template<typename T>
+	Optional<T>::Optional(const Optional<T>& other)
+		: m_has_value(other.has_value())
+	{
+		if (other.has_value())
+			new (m_storage) T(other.value());
+	}
+
 	template<typename T>
 	Optional<T>::Optional(const T& value)
 		: m_has_value(true)
@@ -61,7 +79,7 @@ namespace BAN
 	Optional<T>::Optional(T&& value)
 		: m_has_value(true)
 	{
-		new (m_storage) T(BAN::move(value));
+		new (m_storage) T(move(value));
 	}
 
 	template<typename T>
@@ -69,7 +87,7 @@ namespace BAN
 	Optional<T>::Optional(Args&&... args)
 		: m_has_value(true)
 	{
-		new (m_storage) T(BAN::forward<Args>(args)...);
+		new (m_storage) T(forward<Args>(args)...);
 	}
 
 	template<typename T>
@@ -79,26 +97,22 @@ namespace BAN
 	}
 
 	template<typename T>
-	Optional<T>& Optional<T>::operator=(const Optional& other)
+	Optional<T>& Optional<T>::operator=(Optional&& other)
 	{
 		clear();
+		m_has_value = other.has_value();
 		if (other.has_value())
-		{
-			m_has_value = true;
-			new (m_storage) T(other.value());
-		}
+			new (m_storage) T(move(other.release_value()));
 		return *this;
 	}
 
 	template<typename T>
-	Optional<T>& Optional<T>::operator=(Optional&& other)
+	Optional<T>& Optional<T>::operator=(const Optional& other)
 	{
 		clear();
-		if (other.has_value())
-		{
-			m_has_value = true;
-			new (m_storage) T(BAN::move(other.release_value()));
-		}
+		m_has_value = other.has_value();
+		if (other.has_value)
+			new (m_storage) T(other.value());
 		return *this;
 	}
 
@@ -108,7 +122,7 @@ namespace BAN
 	{
 		clear();
 		m_has_value = true;
-		new (m_storage) T(BAN::forward<Args>(args)...);
+		new (m_storage) T(forward<Args>(args)...);
 		return *this;
 	}
 
@@ -147,18 +161,13 @@ namespace BAN
 	}
 
 	template<typename T>
-	T&& Optional<T>::release_value()
+	T Optional<T>::release_value()
 	{
 		ASSERT(has_value());
+		T released_value = move(value());
+		value().~T();
 		m_has_value = false;
-		return BAN::move((T&)m_storage);
-	}
-
-	template<typename T>
-	const T& Optional<T>::value() const
-	{
-		ASSERT(has_value());
-		return (const T&)m_storage;
+		return move(released_value);
 	}
 
 	template<typename T>
@@ -168,6 +177,13 @@ namespace BAN
 		return (T&)m_storage;
 	}
 
+	template<typename T>
+	const T& Optional<T>::value() const
+	{
+		ASSERT(has_value());
+		return (const T&)m_storage;
+	}
+
 	template<typename T>
 	void Optional<T>::clear()
 	{

kernel/CMakeLists.txt

@@ -53,6 +53,8 @@ set(KERNEL_SOURCES
 	kernel/Semaphore.cpp
 	kernel/SpinLock.cpp
 	kernel/SSP.cpp
+	kernel/Storage/ATA/AHCI/Controller.cpp
+	kernel/Storage/ATA/AHCI/Device.cpp
 	kernel/Storage/ATA/ATABus.cpp
 	kernel/Storage/ATA/ATAController.cpp
 	kernel/Storage/ATA/ATADevice.cpp

kernel/include/kernel/InterruptController.h

@@ -35,6 +35,8 @@ namespace Kernel
 		static void initialize(bool force_pic);
 		static InterruptController& get();
 
+		bool is_using_apic() const { return m_using_apic; }
+
 		void enter_acpi_mode();
 
 	private:

kernel/include/kernel/Memory/DMARegion.h

@@ -8,13 +8,16 @@ namespace Kernel
 	class DMARegion
 	{
 	public:
-		BAN::ErrorOr<BAN::UniqPtr<DMARegion>> create(size_t size);
+		static BAN::ErrorOr<BAN::UniqPtr<DMARegion>> create(size_t size);
 		~DMARegion();
 
 		size_t size() const { return m_size; }
 		vaddr_t vaddr() const { return m_vaddr; }
 		paddr_t paddr() const { return m_paddr; }
 
+		paddr_t vaddr_to_paddr(vaddr_t vaddr) const { return vaddr - m_vaddr + m_paddr; }
+		vaddr_t paddr_to_vaddr(paddr_t paddr) const { return paddr - m_paddr + m_vaddr; }
+
 	private:
 		DMARegion(size_t size, vaddr_t vaddr, paddr_t paddr);
 

kernel/include/kernel/PCI.h

@@ -75,6 +75,8 @@ namespace Kernel::PCI
 
 		uint8_t header_type() const { return m_header_type; }
 
+		BAN::ErrorOr<uint8_t> get_irq();
+
 		BAN::ErrorOr<BAN::UniqPtr<BarRegion>> allocate_bar_region(uint8_t bar_num);
 
 		void enable_bus_mastering();
@@ -92,16 +94,22 @@ namespace Kernel::PCI
 	private:
 		void enumerate_capabilites();
 
+		void set_command_bits(uint16_t mask);
+		void unset_command_bits(uint16_t mask);
+
 	private:
-		uint8_t m_bus;
-		uint8_t m_dev;
-		uint8_t m_func;
+		const uint8_t m_bus;
+		const uint8_t m_dev;
+		const uint8_t m_func;
 
 		uint8_t m_class_code;
 		uint8_t m_subclass;
 		uint8_t m_prog_if;
 
 		uint8_t m_header_type;
+
+		BAN::Optional<uint8_t> m_offset_msi;
+		BAN::Optional<uint8_t> m_offset_msi_x;
 	};
 
 	class PCIManager

kernel/include/kernel/Storage/ATA/AHCI/Controller.h

@@ -0,0 +1,43 @@
+#pragma once
+
+#include <BAN/Array.h>
+#include <BAN/RefPtr.h>
+#include <kernel/InterruptController.h>
+#include <kernel/Memory/DMARegion.h>
+#include <kernel/PCI.h>
+#include <kernel/Storage/ATA/AHCI/Definitions.h>
+
+namespace Kernel
+{
+
+	class AHCIController final : public StorageController, public Interruptable
+	{
+		BAN_NON_COPYABLE(AHCIController);
+		BAN_NON_MOVABLE(AHCIController);
+
+	public:
+		~AHCIController();
+
+		virtual void handle_irq() override;
+
+		uint32_t command_slot_count() const { return m_command_slot_count; }
+
+	private:
+		AHCIController(PCI::Device& pci_device)
+			: m_pci_device(pci_device)
+		{ }
+		BAN::ErrorOr<void> initialize();
+		BAN::Optional<AHCIPortType> check_port_type(volatile HBAPortMemorySpace&);
+
+	private:
+		PCI::Device& m_pci_device;
+		BAN::UniqPtr<PCI::BarRegion> m_abar;
+
+		BAN::Array<AHCIDevice*, 32> m_devices;
+
+		uint32_t m_command_slot_count { 0 };
+
+		friend class ATAController;
+	};
+
+}

kernel/include/kernel/Storage/ATA/AHCI/Definitions.h

@@ -0,0 +1,294 @@
+#pragma once
+
+#include <stdint.h>
+
+#define FIS_TYPE_REGISTER_H2D		0x27
+#define FIS_TYPE_REGISTER_D2H		0x34
+#define FIS_TYPE_DMA_ACT			0x39
+#define FIS_TYPE_DMA_SETUP			0x41
+#define FIS_TYPE_DATA				0x46
+#define FIS_TYPE_BIST				0x58
+#define FIS_TYPE_PIO_SETUP			0x5F
+#define FIS_TYPE_SET_DEVIVE_BITS	0xA1
+
+#define SATA_CAP_SUPPORTS64	(1 << 31)
+
+#define SATA_GHC_AHCI_ENABLE		(1 << 31)
+#define SATA_GHC_INTERRUPT_ENABLE	(1 << 1)
+
+#define	SATA_SIG_ATA	0x00000101
+#define	SATA_SIG_ATAPI	0xEB140101
+#define	SATA_SIG_SEMB	0xC33C0101
+#define	SATA_SIG_PM		0x96690101
+
+#define HBA_PORT_IPM_ACTIVE		1
+#define HBA_PORT_DET_PRESENT	3
+
+#define HBA_PxCMD_ST	0x0001
+#define HBA_PxCMD_FRE	0x0010
+#define HBA_PxCMD_FR	0x4000
+#define HBA_PxCMD_CR	0x8000
+
+namespace Kernel
+{
+
+	static constexpr uint32_t s_hba_prdt_count { 8 };
+
+	struct FISRegisterH2D
+	{
+		uint8_t  fis_type;			// FIS_TYPE_REGISTER_H2D
+	
+		uint8_t  pm_port : 4;		// Port multiplier
+		uint8_t  __reserved0 : 3;
+		uint8_t  c : 1;				// 1: Command, 0: Control
+	
+		uint8_t  command;
+		uint8_t  feature_lo;		// Feature register, 7:0
+	
+		uint8_t  lba0;				// LBA low register, 7:0
+		uint8_t  lba1;				// LBA mid register, 15:8
+		uint8_t  lba2;				// LBA high register, 23:16
+		uint8_t  device;
+	
+		uint8_t  lba3;				// LBA register, 31:24
+		uint8_t  lba4;				// LBA register, 39:32
+		uint8_t  lba5;				// LBA register, 47:40
+		uint8_t  feature_hi;		// Feature register, 15:8
+	
+		uint8_t  count_lo;			// Count register, 7:0
+		uint8_t  count_hi;			// Count register, 15:8
+		uint8_t  icc;				// Isochronous command completion
+		uint8_t  control;
+	
+		uint8_t  __reserved1[4];
+	} __attribute__((packed));
+
+	struct FISRegisterD2H
+	{
+		uint8_t  fis_type;			// FIS_TYPE_REGISTER_D2H
+	
+		uint8_t  pm_port : 4;		// Port multiplier
+		uint8_t  __reserved0 : 2;
+		uint8_t  i : 1;				// Interrupt bit
+		uint8_t  __reserved1 : 1;
+	
+		uint8_t  status;
+		uint8_t  error;
+	
+		uint8_t  lba0;				// LBA low register, 7:0
+		uint8_t  lba1;				// LBA mid register, 15:8
+		uint8_t  lba2;				// LBA high register, 23:16
+		uint8_t  device;
+	
+		uint8_t  lba3;				// LBA register, 31:24
+		uint8_t  lba4;				// LBA register, 39:32
+		uint8_t  lba5;				// LBA register, 47:40
+		uint8_t  __reserved2;
+	
+		uint8_t  count_lo;			// Count register, 7:0
+		uint8_t  count_hi;			// Count register, 15:8
+		uint8_t  __reserved3[2];
+	
+		uint8_t  __reserved4[4];
+	} __attribute__((packed));
+
+	struct FISDataBI
+	{
+		uint8_t  fis_type;			// FIS_TYPE_DATA
+	
+		uint8_t  pm_port : 4;		// Port multiplier
+		uint8_t  __reserved0 : 4;
+	
+		uint8_t  __reserved1[2];
+
+		uint32_t data[0];			// Payload (1 - 2048 dwords)
+	} __attribute__((packed));
+
+	struct SetDeviceBitsD2H
+	{
+		uint8_t fis_type;		// FIS_TYPE_SET_DEVICE_BITS
+
+		uint8_t pm_port : 4;	// Port multiplier
+		uint8_t __reserved0 : 2;
+		uint8_t i : 1;			// Interrupt bit
+		uint8_t n : 1;			// Notification bit
+
+		uint8_t status;
+		uint8_t error;
+
+		uint32_t __reserved1;
+	} __attribute__((packed));
+
+	struct PIOSetupD2H
+	{
+		uint8_t  fis_type;			// FIS_TYPE_PIO_SETUP
+	
+		uint8_t  pm_port : 4;		// Port multiplier
+		uint8_t  __reserved0 : 1;
+		uint8_t  d : 1;				// Data transfer direction, 1 - device to host
+		uint8_t  i : 1;				// Interrupt bit
+		uint8_t  __reserved1 : 1;
+	
+		uint8_t  status;
+		uint8_t  error;
+	
+		uint8_t  lba0;				// LBA low register, 7:0
+		uint8_t  lba1;				// LBA mid register, 15:8
+		uint8_t  lba2;				// LBA high register, 23:16
+		uint8_t  device;
+	
+		uint8_t  lba3;				// LBA register, 31:24
+		uint8_t  lba4;				// LBA register, 39:32
+		uint8_t  lba5;				// LBA register, 47:40
+		uint8_t  __reserved2;
+	
+		uint8_t  count_lo;			// Count register, 7:0
+		uint8_t  count_hi;			// Count register, 15:8
+		uint8_t  __reserved3;
+		uint8_t  e_status;			// New value of status register
+	
+		uint16_t tc;				// Transfer count
+		uint8_t  __reserved4[2];
+	} __attribute__((packed));
+
+	struct DMASetupBI
+	{
+		uint8_t  fis_type;				// FIS_TYPE_DMA_SETUP
+	
+		uint8_t  pm_port : 4;			// Port multiplier
+		uint8_t  __reserved0 : 1;
+		uint8_t  d : 1;					// Data transfer direction, 1 - device to host
+		uint8_t  i : 1;					// Interrupt bit
+		uint8_t  a : 1;					// Auto-activate. Specifies if DMA Activate FIS is needed
+	
+		uint8_t  __reserved1[2];
+
+		uint64_t dma_buffer_id;			// DMA Buffer Identifier. Used to Identify DMA buffer in host memory.
+										// SATA Spec says host specific and not in Spec. Trying AHCI spec might work.
+
+		uint32_t __reserved2;
+
+		uint32_t dma_buffer_offset;		// Byte offset into buffer. First 2 bits must be 0
+
+		uint32_t dma_transfer_count;	// Number of bytes to transfer. Bit 0 must be 0
+
+		uint32_t __reserved3;
+	} __attribute__((packed));
+
+	struct HBAPortMemorySpace
+	{
+		uint32_t clb;		// command list base address, 1K-byte aligned
+		uint32_t clbu;		// command list base address upper 32 bits
+		uint32_t fb;		// FIS base address, 256-byte aligned
+		uint32_t fbu;		// FIS base address upper 32 bits
+		uint32_t is;		// interrupt status
+		uint32_t ie;		// interrupt enable
+		uint32_t cmd;		// command and status
+		uint32_t __reserved0;
+		uint32_t tfd;		// task file data
+		uint32_t sig;		// signature
+		uint32_t ssts;		// SATA status (SCR0:SStatus)
+		uint32_t sctl;		// SATA control (SCR2:SControl)
+		uint32_t serr;		// SATA error (SCR1:SError)
+		uint32_t sact;		// SATA active (SCR3:SActive)
+		uint32_t ci;		// command issue
+		uint32_t sntf;		// SATA notification (SCR4:SNotification)
+		uint32_t fbs;		// FIS-based switch control
+		uint32_t __reserved1[11];
+		uint32_t vendor[4];
+	} __attribute__((packed));
+
+	struct HBAGeneralMemorySpace
+	{
+		uint32_t cap;		// Host capability
+		uint32_t ghc;		// Global host control
+		uint32_t is;		// Interrupt status
+		uint32_t pi;		// Port implemented
+		uint32_t vs;		// Version
+		uint32_t ccc_ctl;	// Command completion coalescing control
+		uint32_t ccc_pts;	// Command completion coalescing ports
+		uint32_t em_loc;	// 0x1C, Enclosure management location
+		uint32_t em_ctl;	// 0x20, Enclosure management control
+		uint32_t cap2;		// 0x24, Host capabilities extended
+		uint32_t bohc;		// 0x28, BIOS/OS handoff control and status
+	
+		uint8_t  __reserved0[0xA0-0x2C];
+	
+		uint8_t  vendor[0x100-0xA0];
+	
+		HBAPortMemorySpace ports[0]; // 1 - 32 ports
+	} __attribute__((packed));
+
+	struct ReceivedFIS
+	{
+		DMASetupBI			dsfis;
+		uint8_t				pad0[4];
+	
+		PIOSetupD2H			psfis;
+		uint8_t				pad1[12];
+	
+		FISRegisterD2H		rfis;
+		uint8_t				pad2[4];
+	
+		SetDeviceBitsD2H	sdbfis;
+	
+		uint8_t				ufis[64];
+	
+		uint8_t				__reserved[0x100-0xA0];
+	} __attribute__((packed));
+
+	struct HBACommandHeader
+	{
+		uint8_t  cfl : 5;	// Command FIS length in DWORDS, 2 ~ 16
+		uint8_t  a : 1;		// ATAPI
+		uint8_t  w : 1;		// Write, 1: H2D, 0: D2H
+		uint8_t  p : 1;		// Prefetchable
+	
+		uint8_t  r : 1;		// Reset
+		uint8_t  b : 1;		// BIST
+		uint8_t  c : 1;		// Clear busy upon R_OK
+		uint8_t  __reserved0 : 1;
+		uint8_t  pmp : 4;	// Port multiplier port
+	
+		uint16_t prdtl;		// Physical region descriptor table length in entries
+	
+		volatile uint32_t prdbc;	// Physical region descriptor byte count transferred
+	
+		uint32_t ctba;		// Command table descriptor base address
+		uint32_t ctbau;		// Command table descriptor base address upper 32 bits
+	
+		uint32_t __reserved1[4];
+	} __attribute__((packed));
+
+	struct HBAPRDTEntry
+	{
+		uint32_t dba;			// Data base address
+		uint32_t dbau;			// Data base address upper 32 bits
+		uint32_t __reserved0;
+
+		uint32_t dbc : 22;		// Byte count, 4M max
+		uint32_t __reserved1 : 9;
+		uint32_t i : 1;			// Interrupt on completion
+	} __attribute__((packed));
+
+	struct HBACommandTable
+	{
+		uint8_t cfis[64];
+		uint8_t acmd[16];
+		uint8_t __reserved[48];
+		HBAPRDTEntry prdt_entry[s_hba_prdt_count];
+	} __attribute__((packed));
+
+	enum class AHCIPortType
+	{
+		NONE,
+		SATA,
+		SATAPI,
+		SEMB,
+		PM
+	};
+
+	class AHCIController;
+	class AHCIDevice;
+
+}

kernel/include/kernel/Storage/ATA/AHCI/Device.h

@@ -0,0 +1,49 @@
+#pragma once
+
+#include <kernel/Semaphore.h>
+#include <kernel/Storage/ATA/AHCI/Definitions.h>
+#include <kernel/Storage/ATA/ATADevice.h>
+
+namespace Kernel
+{
+
+	class AHCIDevice final : public detail::ATABaseDevice
+	{
+	public:
+		static BAN::ErrorOr<BAN::RefPtr<AHCIDevice>> create(BAN::RefPtr<AHCIController>, volatile HBAPortMemorySpace*);
+		~AHCIDevice() = default;
+
+	private:
+		AHCIDevice(BAN::RefPtr<AHCIController> controller, volatile HBAPortMemorySpace* port)
+			: m_controller(controller)
+			, m_port(port)
+		{ }
+		BAN::ErrorOr<void> initialize();
+		BAN::ErrorOr<void> allocate_buffers();
+		BAN::ErrorOr<void> rebase();
+		BAN::ErrorOr<void> read_identify_data();
+
+		virtual BAN::ErrorOr<void> read_sectors_impl(uint64_t lba, uint64_t sector_count, uint8_t* buffer) override;
+		virtual BAN::ErrorOr<void> write_sectors_impl(uint64_t lba, uint64_t sector_count, const uint8_t* buffer) override;
+		BAN::ErrorOr<void> send_command_and_block(uint64_t lba, uint64_t sector_count, Command command);
+
+		BAN::Optional<uint32_t> find_free_command_slot();
+
+		void handle_irq();
+		void block_until_irq();
+
+	private:
+		BAN::RefPtr<AHCIController> m_controller;
+		volatile HBAPortMemorySpace* const m_port;
+		
+		BAN::UniqPtr<DMARegion> m_dma_region;
+		// Intermediate read/write buffer
+		// TODO: can we read straight to user buffer?
+		BAN::UniqPtr<DMARegion> m_data_dma_region;
+
+		volatile bool m_has_got_irq { false };
+
+		friend class AHCIController;
+	};
+
+}

kernel/include/kernel/Storage/ATA/ATABus.h

@@ -27,8 +27,6 @@ namespace Kernel
 
 		virtual void handle_irq() override;
 
-		void initialize_devfs();
-
 	private:
 		ATABus(uint16_t base, uint16_t ctrl)
 			: m_base(base)
@@ -54,7 +52,7 @@ namespace Kernel
 		const uint16_t m_ctrl;
 		SpinLock m_lock;
 
-		bool m_has_got_irq { false };
+		volatile bool m_has_got_irq { false };
 
 		// Non-owning pointers
 		BAN::Vector<ATADevice*> m_devices;

kernel/include/kernel/Storage/ATA/ATAController.h

@@ -12,7 +12,7 @@ namespace Kernel
 	class ATAController : public StorageController
 	{
 	public:
-		static BAN::ErrorOr<BAN::UniqPtr<StorageController>> create(PCI::Device&);
+		static BAN::ErrorOr<BAN::RefPtr<StorageController>> create(PCI::Device&);
 		virtual BAN::ErrorOr<void> initialize() override;
 
 	private:

kernel/include/kernel/Storage/ATA/ATADefinitions.h

@@ -33,7 +33,11 @@
 #define ATA_STATUS_BSY				0x80
 
 #define ATA_COMMAND_READ_SECTORS	0x20
+#define ATA_COMMAND_READ_DMA		0xC8
+#define ATA_COMMAND_READ_DMA_EXT	0x25
 #define ATA_COMMAND_WRITE_SECTORS	0x30
+#define ATA_COMMAND_WRITE_DMA		0xCA
+#define ATA_COMMAND_WRITE_DMA_EXT	0x35
 #define ATA_COMMAND_IDENTIFY_PACKET	0xA1
 #define ATA_COMMAND_CACHE_FLUSH		0xE7
 #define ATA_COMMAND_IDENTIFY		0xEC

kernel/include/kernel/Storage/ATA/ATADevice.h

@@ -6,52 +6,69 @@
 namespace Kernel
 {
 
-	class ATADevice final : public StorageDevice
+	namespace detail
+	{
+
+		class ATABaseDevice : public StorageDevice
+		{
+		public:
+			enum class Command
+			{
+				Read,
+				Write
+			};
+
+		public:
+			virtual ~ATABaseDevice() {};
+
+			virtual uint32_t sector_size() const override { return m_sector_words * 2; }
+			virtual uint64_t total_size() const override { return m_lba_count * sector_size(); }
+
+			uint32_t words_per_sector() const { return m_sector_words; }
+			uint64_t sector_count() const { return m_lba_count; }
+
+			BAN::StringView model() const { return m_model; }
+			BAN::StringView name() const;
+
+			virtual dev_t rdev() const override { return m_rdev; }
+
+			virtual BAN::ErrorOr<size_t> read_impl(off_t, void*, size_t) override;
+			virtual BAN::ErrorOr<size_t> write_impl(off_t, const void*, size_t) override;
+
+		protected:
+			ATABaseDevice();
+			BAN::ErrorOr<void> initialize(BAN::Span<const uint16_t> identify_data);
+
+		protected:			
+			uint16_t m_signature;
+			uint16_t m_capabilities;
+			uint32_t m_command_set;
+			uint32_t m_sector_words;
+			uint64_t m_lba_count;
+			char m_model[41];
+
+			const dev_t m_rdev;
+		};
+
+	}
+
+	class ATADevice final : public detail::ATABaseDevice
 	{
 	public:
 		static BAN::ErrorOr<BAN::RefPtr<ATADevice>> create(BAN::RefPtr<ATABus>, ATABus::DeviceType, bool is_secondary, BAN::Span<const uint16_t> identify_data);
 
-		virtual uint32_t sector_size() const override { return m_sector_words * 2; }
-		virtual uint64_t total_size() const override { return m_lba_count * sector_size(); }
-
 		bool is_secondary() const { return m_is_secondary; }
-		uint32_t words_per_sector() const { return m_sector_words; }
-		uint64_t sector_count() const { return m_lba_count; }
-
-		BAN::StringView model() const { return m_model; }
-		BAN::StringView name() const;
-
-	protected:
-		virtual BAN::ErrorOr<void> read_sectors_impl(uint64_t, uint8_t, uint8_t*) override;
-		virtual BAN::ErrorOr<void> write_sectors_impl(uint64_t, uint8_t, const uint8_t*) override;
 
 	private:
 		ATADevice(BAN::RefPtr<ATABus>, ATABus::DeviceType, bool is_secodary);
-		BAN::ErrorOr<void> initialize(BAN::Span<const uint16_t> identify_data);
+		
+		virtual BAN::ErrorOr<void> read_sectors_impl(uint64_t, uint64_t, uint8_t*) override;
+		virtual BAN::ErrorOr<void> write_sectors_impl(uint64_t, uint64_t, const uint8_t*) override;
 
 	private:
 		BAN::RefPtr<ATABus> m_bus;
 		const ATABus::DeviceType m_type;
 		const bool m_is_secondary;
-		
-		uint16_t m_signature;
-		uint16_t m_capabilities;
-		uint32_t m_command_set;
-		uint32_t m_sector_words;
-		uint64_t m_lba_count;
-		char m_model[41];
-
-	public:
-		virtual Mode mode() const override { return { Mode::IFBLK | Mode::IRUSR | Mode::IRGRP }; }
-		virtual uid_t uid() const override { return 0; }
-		virtual gid_t gid() const override { return 0; }
-		virtual dev_t rdev() const override { return m_rdev; }
-
-	private:
-		virtual BAN::ErrorOr<size_t> read_impl(off_t, void*, size_t) override;
-
-	public:
-		const dev_t m_rdev;
 	};
 
-}
+}

kernel/include/kernel/Storage/StorageController.h

@@ -1,9 +1,11 @@
 #pragma once
 
+#include <BAN/RefPtr.h>
+
 namespace Kernel
 {
 
-	class StorageController
+	class StorageController : public BAN::RefCounted<StorageController>
 	{
 	public:
 		virtual ~StorageController() {}

kernel/include/kernel/Storage/StorageDevice.h

@@ -73,8 +73,8 @@ namespace Kernel
 
 		BAN::ErrorOr<void> initialize_partitions();
 
-		BAN::ErrorOr<void> read_sectors(uint64_t lba, uint8_t sector_count, uint8_t* buffer);
-		BAN::ErrorOr<void> write_sectors(uint64_t lba, uint8_t sector_count, const uint8_t* buffer);
+		BAN::ErrorOr<void> read_sectors(uint64_t lba, uint64_t sector_count, uint8_t* buffer);
+		BAN::ErrorOr<void> write_sectors(uint64_t lba, uint64_t sector_count, const uint8_t* buffer);
 
 		virtual uint32_t sector_size() const = 0;
 		virtual uint64_t total_size() const = 0;
@@ -86,8 +86,8 @@ namespace Kernel
 		virtual bool is_storage_device() const override { return true; }
 
 	protected:
-		virtual BAN::ErrorOr<void> read_sectors_impl(uint64_t lba, uint8_t sector_count, uint8_t* buffer) = 0;
-		virtual BAN::ErrorOr<void> write_sectors_impl(uint64_t lba, uint8_t sector_count, const uint8_t* buffer) = 0;
+		virtual BAN::ErrorOr<void> read_sectors_impl(uint64_t lba, uint64_t sector_count, uint8_t* buffer) = 0;
+		virtual BAN::ErrorOr<void> write_sectors_impl(uint64_t lba, uint64_t sector_count, const uint8_t* buffer) = 0;
 		void add_disk_cache();
 
 	private:

kernel/include/kernel/Timer/HPET.h

@@ -12,6 +12,7 @@ namespace Kernel
 		static BAN::ErrorOr<BAN::UniqPtr<HPET>> create(bool force_pic);
 
 		virtual uint64_t ms_since_boot() const override;
+		virtual uint64_t ns_since_boot() const override;
 		virtual timespec time_since_boot() const override;
 
 		virtual void handle_irq() override;

kernel/include/kernel/Timer/PIT.h

@@ -12,6 +12,7 @@ namespace Kernel
 		static BAN::ErrorOr<BAN::UniqPtr<PIT>> create();
 
 		virtual uint64_t ms_since_boot() const override;
+		virtual uint64_t ns_since_boot() const override;
 		virtual timespec time_since_boot() const override;
 
 		virtual void handle_irq() override;

kernel/include/kernel/Timer/Timer.h

@@ -14,6 +14,7 @@ namespace Kernel
 	public:
 		virtual ~Timer() {};
 		virtual uint64_t ms_since_boot() const = 0;
+		virtual uint64_t ns_since_boot() const = 0;
 		virtual timespec time_since_boot() const = 0;
 	};
 
@@ -25,6 +26,7 @@ namespace Kernel
 		static bool is_initialized();
 
 		virtual uint64_t ms_since_boot() const override;
+		virtual uint64_t ns_since_boot() const override;
 		virtual timespec time_since_boot() const override;
 		
 		void sleep(uint64_t ms) const;

kernel/kernel/Memory/DMARegion.cpp

@@ -26,7 +26,7 @@ namespace Kernel
 		vaddr_guard.disable();
 		paddr_guard.disable();
 
-		PageTable::kernel().map_range_at(paddr, vaddr, size, PageTable::Flags::CacheDisable | PageTable::Flags::ReadWrite | PageTable::Flags::Reserved);
+		PageTable::kernel().map_range_at(paddr, vaddr, size, PageTable::Flags::CacheDisable | PageTable::Flags::ReadWrite | PageTable::Flags::Present);
 
 		return BAN::UniqPtr<DMARegion>::adopt(region_ptr);
 	}
@@ -40,7 +40,7 @@ namespace Kernel
 	DMARegion::~DMARegion()
 	{
 		PageTable::kernel().unmap_range(m_vaddr, m_size);
-		Heap::get().release_contiguous_pages(m_vaddr, BAN::Math::div_round_up<size_t>(m_size, PAGE_SIZE));
+		Heap::get().release_contiguous_pages(m_paddr, BAN::Math::div_round_up<size_t>(m_size, PAGE_SIZE));
 	}
 
 }

kernel/kernel/Memory/PhysicalRange.cpp

@@ -13,7 +13,7 @@ namespace Kernel
 	PhysicalRange::PhysicalRange(paddr_t paddr, size_t size)
 		: m_paddr(paddr)
 		, m_size(size)
-		, m_bitmap_pages(BAN::Math::div_round_up<size_t>(size / PAGE_SIZE, 8))
+		, m_bitmap_pages(BAN::Math::div_round_up<size_t>(size / PAGE_SIZE, PAGE_SIZE * 8))
 		, m_data_pages((size / PAGE_SIZE) - m_bitmap_pages)
 		, m_free_pages(m_data_pages)
 	{
@@ -26,9 +26,16 @@ namespace Kernel
 		PageTable::kernel().map_range_at(m_paddr, m_vaddr, size, PageTable::Flags::ReadWrite | PageTable::Flags::Present);
 
 		memset((void*)m_vaddr, 0x00, m_bitmap_pages * PAGE_SIZE);
-		memset((void*)m_vaddr, 0xFF, m_data_pages / 8);
-		for (ull i = 0; i < m_data_pages % 8; i++)
-			((uint8_t*)m_vaddr)[m_data_pages / 8] |= 1 << i;
+
+		for (ull i = 0; i < m_data_pages / ull_bits; i++)
+			ull_bitmap_ptr()[i] = ~0ull;
+
+		if (m_data_pages % ull_bits)
+		{
+			ull off = m_data_pages / ull_bits;
+			ull bits = m_data_pages % ull_bits;
+			ull_bitmap_ptr()[off] = ~(~0ull << bits);
+		}
 
 		dprintln("physical range needs {} pages for bitmap", m_bitmap_pages);
 	}

kernel/kernel/PCI.cpp

@@ -6,12 +6,25 @@
 #include <kernel/Storage/ATA/AHCI/Controller.h>
 #include <kernel/Storage/ATA/ATAController.h>
 
+#include <lai/helpers/pci.h>
+
 #define INVALID_VENDOR 0xFFFF
 #define MULTI_FUNCTION 0x80
 
 #define CONFIG_ADDRESS 0xCF8
 #define CONFIG_DATA 0xCFC
 
+#define PCI_REG_COMMAND 0x04
+#define PCI_REG_STATUS 0x06
+#define PCI_REG_CAPABILITIES 0x34
+#define PCI_REG_IRQ_LINE 0x3C
+#define PCI_REG_IRQ_PIN 0x44
+
+#define PCI_CMD_IO_SPACE (1 << 0)
+#define PCI_CMD_MEM_SPACE (1 << 1)
+#define PCI_CMD_BUS_MASTER (1 << 2)
+#define PCI_CMD_INTERRUPT_DISABLE (1 << 10)
+
 #define DEBUG_PCI 1
 
 namespace Kernel::PCI
@@ -179,10 +192,9 @@ namespace Kernel::PCI
 	{
 		ASSERT(device.header_type() == 0x00);
 
-		uint32_t command_status = device.read_dword(0x04);
-
 		// disable io/mem space while reading bar
-		device.write_dword(0x04, command_status & ~3);
+		uint16_t command = device.read_word(PCI_REG_COMMAND);
+		device.write_word(PCI_REG_COMMAND, command & ~(PCI_CMD_IO_SPACE | PCI_CMD_MEM_SPACE));
 
 		uint8_t offset = 0x10 + bar_num * 4;
 
@@ -224,9 +236,9 @@ namespace Kernel::PCI
 		auto region = BAN::UniqPtr<BarRegion>::adopt(region_ptr);
 		TRY(region->initialize());
 
-		// restore old command register and enable correct IO/MEM
-		command_status |= (type == BarType::IO) ? 1 : 2;
-		device.write_dword(0x04, command_status);
+		// restore old command register and enable correct IO/MEM space
+		command |= (type == BarType::IO) ? PCI_CMD_IO_SPACE : PCI_CMD_MEM_SPACE;
+		device.write_word(PCI_REG_COMMAND, command);
 
 #if DEBUG_PCI
 		dprintln("created BAR region for PCI {}:{}.{}",
@@ -365,58 +377,115 @@ namespace Kernel::PCI
 
 	void PCI::Device::enumerate_capabilites()
 	{
-		uint16_t status = read_word(0x06);
+		uint16_t status = read_word(PCI_REG_STATUS);
 		if (!(status & (1 << 4)))
 			return;
 
-		uint8_t capabilities = read_byte(0x34) & 0xFC;
-		while (capabilities)
+		uint8_t capability_offset = read_byte(PCI_REG_CAPABILITIES) & 0xFC;
+		while (capability_offset)
 		{
-			uint16_t next = read_word(capabilities);
-			dprintln("  cap {2H}", next & 0xFF);
-			capabilities = (next >> 8) & 0xFC;
+			uint16_t capability_info = read_word(capability_offset);
+
+			switch (capability_info & 0xFF)
+			{
+				case 0x05:
+					m_offset_msi = capability_offset;
+					dprintln("{}:{}.{} has MSI", m_bus, m_dev, m_func);
+					break;
+				case 0x11:
+					m_offset_msi_x = capability_offset;
+					dprintln("{}:{}.{} has MSI-X", m_bus, m_dev, m_func);
+					break;
+				default:
+					break;
+			}
+
+			capability_offset = (capability_info >> 8) & 0xFC;
 		}
 	}
 
+	BAN::ErrorOr<uint8_t> PCI::Device::get_irq()
+	{
+		// Legacy PIC just uses the interrupt line field
+		if (!InterruptController::get().is_using_apic())
+			return read_byte(PCI_REG_IRQ_LINE);
+		
+		// TODO: use MSI and MSI-X if supported
+
+		if (m_offset_msi.has_value())
+		{
+		}
+
+		if (m_offset_msi_x.has_value())
+		{
+		}
+
+		for (uint8_t irq_pin = 1; irq_pin <= 4; irq_pin++)
+		{
+			acpi_resource_t dest;
+			auto err = lai_pci_route_pin(&dest, 0, m_bus, m_dev, m_func, irq_pin);
+			if (err != LAI_ERROR_NONE)
+			{
+				dprintln("{}", lai_api_error_to_string(err));
+				continue;
+			}
+
+			write_byte(PCI_REG_IRQ_PIN, irq_pin);
+			return dest.base;
+		}
+
+		dwarnln("Could not allocate interrupt for PCI {}:{}.{}", m_bus, m_dev, m_func);
+		return BAN::Error::from_errno(ENOTSUP);
+	}
+
+	void PCI::Device::set_command_bits(uint16_t mask)
+	{
+		write_dword(PCI_REG_COMMAND, read_dword(PCI_REG_COMMAND) | mask);
+	}
+
+	void PCI::Device::unset_command_bits(uint16_t mask)
+	{
+		write_dword(PCI_REG_COMMAND, read_dword(PCI_REG_COMMAND) & ~mask);
+	}
+
 	void PCI::Device::enable_bus_mastering()
 	{
-		write_dword(0x04, read_dword(0x04) | 1u << 2);
+		set_command_bits(PCI_CMD_BUS_MASTER);
 	}
 
 	void PCI::Device::disable_bus_mastering()
 	{
-		write_dword(0x04, read_dword(0x04) & ~(1u << 2));
-
+		unset_command_bits(PCI_CMD_BUS_MASTER);
 	}
 
 	void PCI::Device::enable_memory_space()
 	{
-		write_dword(0x04, read_dword(0x04) | 1u << 1);
+		set_command_bits(PCI_CMD_MEM_SPACE);
 	}
 
 	void PCI::Device::disable_memory_space()
 	{
-		write_dword(0x04, read_dword(0x04) & ~(1u << 1));
+		unset_command_bits(PCI_CMD_MEM_SPACE);
 	}
 
 	void PCI::Device::enable_io_space()
 	{
-		write_dword(0x04, read_dword(0x04) | 1u << 0);
+		set_command_bits(PCI_CMD_IO_SPACE);
 	}
 
 	void PCI::Device::disable_io_space()
 	{
-		write_dword(0x04, read_dword(0x04) & ~(1u << 0));
+		unset_command_bits(PCI_CMD_IO_SPACE);
 	}
 
 	void PCI::Device::enable_pin_interrupts()
 	{
-		write_dword(0x04, read_dword(0x04) | 1u << 10);
+		unset_command_bits(PCI_CMD_INTERRUPT_DISABLE);
 	}
 
 	void PCI::Device::disable_pin_interrupts()
 	{
-		write_dword(0x04, read_dword(0x04) & ~(1u << 10));
+		set_command_bits(PCI_CMD_INTERRUPT_DISABLE);
 	}
 
 }

kernel/kernel/Storage/ATA/AHCI/Controller.cpp

@@ -0,0 +1,120 @@
+#include <kernel/Memory/Heap.h>
+#include <kernel/Memory/PageTable.h>
+#include <kernel/Storage/ATA/AHCI/Controller.h>
+#include <kernel/Storage/ATA/AHCI/Definitions.h>
+#include <kernel/Storage/ATA/AHCI/Device.h>
+
+namespace Kernel
+{
+
+	BAN::ErrorOr<void> AHCIController::initialize()
+	{
+		m_abar = TRY(m_pci_device.allocate_bar_region(5));
+		if (m_abar->type() != PCI::BarType::MEM)
+		{
+			dprintln("ABAR not MMIO");
+			return BAN::Error::from_errno(EINVAL);
+		}
+
+		auto& abar_mem = *(volatile HBAGeneralMemorySpace*)m_abar->vaddr();
+		if (!(abar_mem.ghc & SATA_GHC_AHCI_ENABLE))
+		{
+			dprintln("Controller not in AHCI mode");
+			return BAN::Error::from_errno(EINVAL);
+		}
+
+		// Enable interrupts and bus mastering
+		m_pci_device.enable_bus_mastering();
+		m_pci_device.enable_pin_interrupts();
+		set_irq(TRY(m_pci_device.get_irq()));
+		enable_interrupt();
+		abar_mem.ghc = abar_mem.ghc | SATA_GHC_INTERRUPT_ENABLE;
+
+		m_command_slot_count = ((abar_mem.cap >> 8) & 0x1F) + 1;
+
+		uint32_t pi = abar_mem.pi;
+		for (uint32_t i = 0; i < 32 && pi; i++, pi >>= 1)
+		{
+			// Verify we don't access abar outside of its bounds
+			if (sizeof(HBAGeneralMemorySpace) + i * sizeof(HBAPortMemorySpace) > m_abar->size())
+				break;
+
+			if (!(pi & 1))
+				continue;
+			
+			auto type = check_port_type(abar_mem.ports[i]);
+			if (!type.has_value())
+				continue;
+
+			if (type.value() != AHCIPortType::SATA)
+			{
+				dprintln("Non-SATA devices not supported");
+				continue;
+			}
+
+			auto device = AHCIDevice::create(this, &abar_mem.ports[i]);
+			if (device.is_error())
+			{
+				dprintln("{}", device.error());
+				continue;
+			}
+
+			m_devices[i] = device.value().ptr();
+			if (auto ret = m_devices[i]->initialize(); ret.is_error())
+			{
+				dprintln("{}", ret.error());
+				m_devices[i] = nullptr;
+			}
+		}
+
+		return {};
+	}
+
+	AHCIController::~AHCIController()
+	{
+	}
+
+	void AHCIController::handle_irq()
+	{
+		auto& abar_mem = *(volatile HBAGeneralMemorySpace*)m_abar->vaddr();
+
+		uint32_t is = abar_mem.is;
+		for (uint8_t i = 0; i < 32; i++)
+		{
+			if (is & (1 << i))
+			{
+				ASSERT(m_devices[i]);
+				m_devices[i]->handle_irq();
+			}
+		}
+
+		abar_mem.is = is;
+	}
+
+	BAN::Optional<AHCIPortType> AHCIController::check_port_type(volatile HBAPortMemorySpace& port)
+	{
+		uint32_t ssts = port.ssts;
+		uint8_t ipm = (ssts >> 8) & 0x0F;
+		uint8_t det = (ssts >> 0) & 0x0F;
+
+		if (det != HBA_PORT_DET_PRESENT)
+			return {};
+		if (ipm != HBA_PORT_IPM_ACTIVE)
+			return {};
+
+		switch (port.sig)
+		{
+			case SATA_SIG_ATA:
+				return AHCIPortType::SATA;
+			case SATA_SIG_ATAPI:
+				return AHCIPortType::SATAPI;
+			case SATA_SIG_PM:
+				return AHCIPortType::PM;
+			case SATA_SIG_SEMB:
+				return AHCIPortType::SEMB;
+		}
+
+		return {};
+	}
+
+}

kernel/kernel/Storage/ATA/AHCI/Device.cpp

@@ -0,0 +1,276 @@
+#include <kernel/Storage/ATA/AHCI/Controller.h>
+#include <kernel/Storage/ATA/AHCI/Device.h>
+#include <kernel/Storage/ATA/ATADefinitions.h>
+#include <kernel/Thread.h>
+#include <kernel/Timer/Timer.h>
+
+namespace Kernel
+{
+
+	static void start_cmd(volatile HBAPortMemorySpace* port)
+	{
+		while (port->cmd & HBA_PxCMD_CR)
+			continue;
+		port->cmd = port->cmd | HBA_PxCMD_FRE;
+		port->cmd = port->cmd | HBA_PxCMD_ST; 
+	}
+	
+	static void stop_cmd(volatile HBAPortMemorySpace* port)
+	{
+		port->cmd = port->cmd & ~HBA_PxCMD_ST;
+		port->cmd = port->cmd & ~HBA_PxCMD_FRE;
+		while (port->cmd & (HBA_PxCMD_FR | HBA_PxCMD_CR))
+			continue;
+	}
+
+	BAN::ErrorOr<BAN::RefPtr<AHCIDevice>> AHCIDevice::create(BAN::RefPtr<AHCIController> controller, volatile HBAPortMemorySpace* port)
+	{
+		auto* device_ptr = new AHCIDevice(controller, port);
+		if (device_ptr == nullptr)
+			return BAN::Error::from_errno(ENOMEM);
+		return BAN::RefPtr<AHCIDevice>::adopt(device_ptr);
+	}
+
+	BAN::ErrorOr<void> AHCIDevice::initialize()
+	{
+		TRY(allocate_buffers());
+		TRY(rebase());
+
+		// enable interrupts
+		m_port->ie = 0xFFFFFFFF;
+
+		TRY(read_identify_data());
+		TRY(detail::ATABaseDevice::initialize({ (const uint16_t*)m_data_dma_region->vaddr(), m_data_dma_region->size() }));
+
+		return {};
+	}
+
+	BAN::ErrorOr<void> AHCIDevice::allocate_buffers()
+	{
+		uint32_t command_slot_count = m_controller->command_slot_count();
+		size_t needed_bytes = (sizeof(HBACommandHeader) + sizeof(HBACommandTable)) * command_slot_count + sizeof(ReceivedFIS);
+
+		m_dma_region = TRY(DMARegion::create(needed_bytes));
+		memset((void*)m_dma_region->vaddr(), 0x00, m_dma_region->size());
+
+		m_data_dma_region = TRY(DMARegion::create(PAGE_SIZE));
+		memset((void*)m_data_dma_region->vaddr(), 0x00, m_data_dma_region->size());
+		
+		return {};
+	}
+
+	BAN::ErrorOr<void> AHCIDevice::rebase()
+	{
+		ASSERT(m_dma_region);
+
+		uint32_t command_slot_count = m_controller->command_slot_count();
+
+		stop_cmd(m_port);
+
+		paddr_t fis_paddr = m_dma_region->paddr();
+		m_port->fb = fis_paddr & 0xFFFFFFFF;
+		m_port->fbu = fis_paddr >> 32;
+
+		paddr_t command_list_paddr = fis_paddr + sizeof(ReceivedFIS);
+		m_port->clb = command_list_paddr & 0xFFFFFFFF;
+		m_port->clbu = command_list_paddr >> 32;
+
+		auto* command_headers = (HBACommandHeader*)m_dma_region->paddr_to_vaddr(command_list_paddr);
+		paddr_t command_table_paddr = command_list_paddr + command_slot_count * sizeof(HBACommandHeader);
+		for (uint32_t i = 0; i < command_slot_count; i++)
+		{
+			uint64_t command_table_entry_paddr = command_table_paddr + i * sizeof(HBACommandTable);
+			command_headers[i].prdtl = s_hba_prdt_count;
+			command_headers[i].ctba = command_table_entry_paddr & 0xFFFFFFFF;
+			command_headers[i].ctbau = command_table_entry_paddr >> 32;
+		}
+
+		start_cmd(m_port);
+
+		return {};
+	}
+
+	BAN::ErrorOr<void> AHCIDevice::read_identify_data()
+	{
+		ASSERT(m_data_dma_region);
+
+		m_port->is = ~(uint32_t)0;
+
+		auto slot = find_free_command_slot();
+		ASSERT(slot.has_value());
+
+		auto& command_header = ((HBACommandHeader*)m_dma_region->paddr_to_vaddr(m_port->clb))[slot.value()];
+		command_header.cfl = sizeof(FISRegisterH2D) / sizeof(uint32_t);
+		command_header.w = 0;
+		command_header.prdtl = 1;
+
+		auto& command_table = *(HBACommandTable*)m_dma_region->paddr_to_vaddr(command_header.ctba);
+		memset(&command_table, 0x00, sizeof(HBACommandTable));
+		command_table.prdt_entry[0].dba = m_data_dma_region->paddr();
+		command_table.prdt_entry[0].dbc = 511;
+		command_table.prdt_entry[0].i = 1;
+
+		auto& command = *(FISRegisterH2D*)command_table.cfis;
+		command.fis_type = FIS_TYPE_REGISTER_H2D;
+		command.c = 1;
+		command.command = ATA_COMMAND_IDENTIFY;
+
+		while (m_port->tfd & (ATA_STATUS_BSY | ATA_STATUS_DRQ))
+			continue;
+
+		m_port->ci = 1 << slot.value();
+
+		// FIXME: timeout
+		do { block_until_irq(); } while (m_port->ci & (1 << slot.value()));
+
+		return {};
+	}
+
+	static void print_error(uint16_t error)
+	{
+		dprintln("Disk error:");
+		if (error & (1 << 11))
+			dprintln("  Internal Error");
+		if (error & (1 << 10))
+			dprintln("  Protocol Error");
+		if (error & (1 << 9))
+			dprintln("  Persistent Communication or Data Integrity Error");
+		if (error & (1 << 8))
+			dprintln("  Transient Data Integrity Error");
+		if (error & (1 << 1))
+			dprintln("  Recovered Communications Error");
+		if (error & (1 << 0))
+			dprintln("  Recovered Data Integrity Error");
+	}
+
+	void AHCIDevice::handle_irq()
+	{
+		ASSERT(!m_has_got_irq);
+		uint16_t err = m_port->serr & 0xFFFF;
+		if (err)
+			print_error(err);
+		m_has_got_irq = true;
+	}
+
+	void AHCIDevice::block_until_irq()
+	{
+		while (!__sync_bool_compare_and_swap(&m_has_got_irq, true, false))
+			__builtin_ia32_pause();
+	}
+
+	BAN::ErrorOr<void> AHCIDevice::read_sectors_impl(uint64_t lba, uint64_t sector_count, uint8_t* buffer)
+	{
+		const size_t sectors_per_page = PAGE_SIZE / sector_size();
+		for (uint64_t sector_off = 0; sector_off < sector_count; sector_off += sectors_per_page)
+		{
+			uint64_t to_read = BAN::Math::min<uint64_t>(sector_count - sector_off, sectors_per_page);
+
+			TRY(send_command_and_block(lba + sector_off, to_read, Command::Read));
+			memcpy(buffer + sector_off * sector_size(), (void*)m_data_dma_region->vaddr(), to_read * sector_size());
+		}
+
+		return {};
+	}
+
+	BAN::ErrorOr<void> AHCIDevice::write_sectors_impl(uint64_t lba, uint64_t sector_count, const uint8_t* buffer)
+	{
+		const size_t sectors_per_page = PAGE_SIZE / sector_size();
+		for (uint64_t sector_off = 0; sector_off < sector_count; sector_off += sectors_per_page)
+		{
+			uint64_t to_read = BAN::Math::min<uint64_t>(sector_count - sector_off, sectors_per_page);
+
+			memcpy((void*)m_data_dma_region->vaddr(), buffer + sector_off * sector_size(), to_read * sector_size());
+			TRY(send_command_and_block(lba + sector_off, to_read, Command::Write));
+		}
+
+		return {};
+	}
+
+	BAN::ErrorOr<void> AHCIDevice::send_command_and_block(uint64_t lba, uint64_t sector_count, Command command)
+	{
+		ASSERT(m_dma_region);
+		ASSERT(m_data_dma_region);
+		ASSERT(0 < sector_count && sector_count <= 0xFFFF + 1);
+		
+		ASSERT(sector_count * sector_size() <= m_data_dma_region->size());
+
+		m_port->is = ~(uint32_t)0;
+
+		auto slot = find_free_command_slot();
+		ASSERT(slot.has_value());
+		
+		auto& command_header = ((HBACommandHeader*)m_dma_region->paddr_to_vaddr(m_port->clb))[slot.value()];
+		command_header.cfl = sizeof(FISRegisterH2D) / sizeof(uint32_t);
+		command_header.prdtl = 1;
+		switch (command)
+		{
+			case Command::Read:
+				command_header.w = 0;
+				break;
+			case Command::Write:
+				command_header.w = 1;
+				break;
+			default:
+				ASSERT_NOT_REACHED();
+		}
+
+		auto& command_table = *(HBACommandTable*)m_dma_region->paddr_to_vaddr(command_header.ctba);
+		memset(&command_table, 0x00, sizeof(HBACommandTable));
+		command_table.prdt_entry[0].dba = m_data_dma_region->paddr() & 0xFFFFFFFF;
+		command_table.prdt_entry[0].dbau = m_data_dma_region->paddr() >> 32;
+		command_table.prdt_entry[0].dbc = sector_count * sector_size() - 1;
+		command_table.prdt_entry[0].i = 1;
+
+		auto& fis_command = *(FISRegisterH2D*)command_table.cfis;
+		memset(&fis_command, 0x00, sizeof(FISRegisterH2D));
+		fis_command.fis_type = FIS_TYPE_REGISTER_H2D;
+		fis_command.c = 1;
+
+		bool need_extended = lba >= (1 << 28) || sector_count > 0xFF;
+		ASSERT (!need_extended || (m_command_set & ATA_COMMANDSET_LBA48_SUPPORTED));
+
+		switch (command)
+		{
+			case Command::Read:
+				fis_command.command = need_extended ? ATA_COMMAND_READ_DMA_EXT : ATA_COMMAND_READ_DMA;
+				break;
+			case Command::Write:
+				fis_command.command = need_extended ? ATA_COMMAND_WRITE_DMA_EXT : ATA_COMMAND_WRITE_DMA;
+				break;
+			default:
+				ASSERT_NOT_REACHED();
+		}
+
+		fis_command.lba0 = (lba >>  0) & 0xFF;
+		fis_command.lba1 = (lba >>  8) & 0xFF;
+		fis_command.lba2 = (lba >> 16) & 0xFF;
+		fis_command.device = 1 << 6;	// LBA mode
+	
+		fis_command.lba3 = (lba >> 24) & 0xFF;
+		fis_command.lba4 = (lba >> 32) & 0xFF;
+		fis_command.lba5 = (lba >> 40) & 0xFF;
+	
+		fis_command.count_lo = (sector_count >> 0) & 0xFF;
+		fis_command.count_hi = (sector_count >> 8) & 0xFF;
+
+		while (m_port->tfd & (ATA_STATUS_BSY | ATA_STATUS_DRQ))
+			continue;
+
+		m_port->ci = 1 << slot.value();
+
+		// FIXME: timeout
+		do { block_until_irq(); } while (m_port->ci & (1 << slot.value()));
+
+		return {};
+	}
+
+	BAN::Optional<uint32_t> AHCIDevice::find_free_command_slot()
+	{
+		uint32_t slots = m_port->sact | m_port->ci;
+		for (uint32_t i = 0; i < m_controller->command_slot_count(); i++, slots >>= 1)
+			if (!(slots & 1))
+				return i;
+		return {};
+	}
+
+}

kernel/kernel/Storage/ATA/ATABus.cpp

@@ -41,6 +41,10 @@ namespace Kernel
 			else
 				device_type = res.value();
 
+			// Enable interrupts
+			select_device(is_secondary);
+			io_write(ATA_PORT_CONTROL, 0);
+
 			auto device_or_error = ATADevice::create(this, device_type, is_secondary, identify_buffer.span());
 
 			if (device_or_error.is_error())
@@ -50,35 +54,23 @@ namespace Kernel
 			}
 
 			auto device = device_or_error.release_value();
-			device->ref();
 			TRY(m_devices.push_back(device.ptr()));
 		}
 
-		// Enable disk interrupts
-		for (auto& device : m_devices)
-		{
-			select_device(device->is_secondary());
-			io_write(ATA_PORT_CONTROL, 0);
-		}
-
 		return {};
 	}
 
-	void ATABus::initialize_devfs()
+	static void select_delay()
 	{
-		for (auto& device : m_devices)
-		{
-			DevFileSystem::get().add_device(device);
-			if (auto res = device->initialize_partitions(); res.is_error())
-				dprintln("{}", res.error());
-			device->unref();
-		}
+		auto time = SystemTimer::get().ns_since_boot();
+		while (SystemTimer::get().ns_since_boot() < time + 400)
+			continue;
 	}
 
 	void ATABus::select_device(bool secondary)
 	{
 		io_write(ATA_PORT_DRIVE_SELECT, 0xA0 | ((uint8_t)secondary << 4));
-		SystemTimer::get().sleep(1);
+		select_delay();
 	}
 
 	BAN::ErrorOr<ATABus::DeviceType> ATABus::identify(bool secondary, BAN::Span<uint16_t> buffer)
@@ -236,6 +228,9 @@ namespace Kernel
 		{
 			// LBA28
 			io_write(ATA_PORT_DRIVE_SELECT, 0xE0 | ((uint8_t)device.is_secondary() << 4) | ((lba >> 24) & 0x0F));
+			select_delay();
+			io_write(ATA_PORT_CONTROL, 0);
+
 			io_write(ATA_PORT_SECTOR_COUNT, sector_count);
 			io_write(ATA_PORT_LBA0, (uint8_t)(lba >>  0));
 			io_write(ATA_PORT_LBA1, (uint8_t)(lba >>  8));
@@ -268,14 +263,15 @@ namespace Kernel
 		{
 			// LBA28
 			io_write(ATA_PORT_DRIVE_SELECT, 0xE0 | ((uint8_t)device.is_secondary() << 4) | ((lba >> 24) & 0x0F));
+			select_delay();
+			io_write(ATA_PORT_CONTROL, 0);
+
 			io_write(ATA_PORT_SECTOR_COUNT, sector_count);
 			io_write(ATA_PORT_LBA0, (uint8_t)(lba >>  0));
 			io_write(ATA_PORT_LBA1, (uint8_t)(lba >>  8));
 			io_write(ATA_PORT_LBA2, (uint8_t)(lba >> 16));
 			io_write(ATA_PORT_COMMAND, ATA_COMMAND_WRITE_SECTORS);
 
-			SystemTimer::get().sleep(1);
-
 			for (uint32_t sector = 0; sector < sector_count; sector++)
 			{
 				write_buffer(ATA_PORT_DATA, (uint16_t*)buffer + sector * device.words_per_sector(), device.words_per_sector());

kernel/kernel/Storage/ATA/ATAController.cpp

@@ -1,4 +1,5 @@
 #include <kernel/FS/DevFS/FileSystem.h>
+#include <kernel/Storage/ATA/AHCI/Controller.h>
 #include <kernel/Storage/ATA/ATABus.h>
 #include <kernel/Storage/ATA/ATAController.h>
 #include <kernel/Storage/ATA/ATADefinitions.h>
@@ -7,7 +8,7 @@
 namespace Kernel
 {
 
-	BAN::ErrorOr<BAN::UniqPtr<StorageController>> ATAController::create(PCI::Device& pci_device)
+	BAN::ErrorOr<BAN::RefPtr<StorageController>> ATAController::create(PCI::Device& pci_device)
 	{
 		StorageController* controller_ptr = nullptr;
 
@@ -20,8 +21,8 @@ namespace Kernel
 				dwarnln("unsupported DMA ATA Controller");
 				return BAN::Error::from_errno(ENOTSUP);
 			case 0x06:
-				dwarnln("unsupported SATA Controller");
-				return BAN::Error::from_errno(ENOTSUP);
+				controller_ptr = new AHCIController(pci_device);
+				break;
 			default:
 				ASSERT_NOT_REACHED();
 		}
@@ -29,7 +30,7 @@ namespace Kernel
 		if (controller_ptr == nullptr)
 			return BAN::Error::from_errno(ENOMEM);
 
-		auto controller = BAN::UniqPtr<StorageController>::adopt(controller_ptr);
+		auto controller = BAN::RefPtr<StorageController>::adopt(controller_ptr);
 		TRY(controller->initialize());
 		return controller;
 	}
@@ -67,9 +68,6 @@ namespace Kernel
 			dprintln("unsupported IDE ATABus in native mode");
 		}
 
-		for (auto& bus : buses)
-			bus->initialize_devfs();
-
 		return {};
 	}
 

kernel/kernel/Storage/ATA/ATADevice.cpp

@@ -21,24 +21,11 @@ namespace Kernel
 		return minor++;
 	}
 
-	BAN::ErrorOr<BAN::RefPtr<ATADevice>> ATADevice::create(BAN::RefPtr<ATABus> bus, ATABus::DeviceType type, bool is_secondary, BAN::Span<const uint16_t> identify_data)
-	{
-		auto* device_ptr = new ATADevice(bus, type, is_secondary);
-		if (device_ptr == nullptr)
-			return BAN::Error::from_errno(ENOMEM);
-		auto device = BAN::RefPtr<ATADevice>::adopt(device_ptr);
-		TRY(device->initialize(identify_data));
-		return device;
-	}
-
-	ATADevice::ATADevice(BAN::RefPtr<ATABus> bus, ATABus::DeviceType type, bool is_secondary)
-		: m_bus(bus)
-		, m_type(type)
-		, m_is_secondary(is_secondary)
-		, m_rdev(makedev(get_ata_dev_major(), get_ata_dev_minor()))
+	detail::ATABaseDevice::ATABaseDevice()
+		: m_rdev(makedev(get_ata_dev_major(), get_ata_dev_minor()))
 	{ }
 
-	BAN::ErrorOr<void> ATADevice::initialize(BAN::Span<const uint16_t> identify_data)
+	BAN::ErrorOr<void> detail::ATABaseDevice::initialize(BAN::Span<const uint16_t> identify_data)
 	{
 		ASSERT(identify_data.size() >= 256);
 
@@ -77,41 +64,74 @@ namespace Kernel
 		}
 		m_model[40] = 0;
 
-		dprintln("ATA disk {} MB", total_size() / 1024 / 1024);
+		size_t model_len = 40;
+		while (model_len > 0 && m_model[model_len - 1] == ' ')
+			model_len--;
+
+		dprintln("Initialized disk '{}' {} MB", BAN::StringView(m_model, model_len), total_size() / 1024 / 1024);
 
 		add_disk_cache();
 
+		DevFileSystem::get().add_device(this);
+		if (auto res = initialize_partitions(); res.is_error())
+			dprintln("{}", res.error());
+
 		return {};
 	}
 
-	BAN::ErrorOr<void> ATADevice::read_sectors_impl(uint64_t lba, uint8_t sector_count, uint8_t* buffer)
+	BAN::ErrorOr<size_t> detail::ATABaseDevice::read_impl(off_t offset, void* buffer, size_t bytes)
 	{
-		TRY(m_bus->read(*this, lba, sector_count, buffer));
-		return {};
-	}
-
-	BAN::ErrorOr<void> ATADevice::write_sectors_impl(uint64_t lba, uint8_t sector_count, const uint8_t* buffer)
-	{
-		TRY(m_bus->write(*this, lba, sector_count, buffer));
-		return {};
-	}
-
-	BAN::ErrorOr<size_t> ATADevice::read_impl(off_t offset, void* buffer, size_t bytes)
-	{
-		ASSERT(offset >= 0);
-		if (offset % sector_size() || bytes % sector_size())
+		if (offset % sector_size())
+			return BAN::Error::from_errno(EINVAL);
+		if (bytes % sector_size())
 			return BAN::Error::from_errno(EINVAL);
-		if ((size_t)offset == total_size())
-			return 0;
 		TRY(read_sectors(offset / sector_size(), bytes / sector_size(), (uint8_t*)buffer));
 		return bytes;
 	}
 
-	BAN::StringView ATADevice::name() const
+	BAN::ErrorOr<size_t> detail::ATABaseDevice::write_impl(off_t offset, const void* buffer, size_t bytes)
+	{
+		if (offset % sector_size())
+			return BAN::Error::from_errno(EINVAL);
+		if (bytes % sector_size())
+			return BAN::Error::from_errno(EINVAL);
+		TRY(write_sectors(offset / sector_size(), bytes / sector_size(), (const uint8_t*)buffer));
+		return bytes;
+	}
+
+	BAN::StringView detail::ATABaseDevice::name() const
 	{
 		static char device_name[] = "sda";
 		device_name[2] += minor(m_rdev);
 		return device_name;
 	}
 
+	BAN::ErrorOr<BAN::RefPtr<ATADevice>> ATADevice::create(BAN::RefPtr<ATABus> bus, ATABus::DeviceType type, bool is_secondary, BAN::Span<const uint16_t> identify_data)
+	{
+		auto* device_ptr = new ATADevice(bus, type, is_secondary);
+		if (device_ptr == nullptr)
+			return BAN::Error::from_errno(ENOMEM);
+		auto device = BAN::RefPtr<ATADevice>::adopt(device_ptr);
+		TRY(device->initialize(identify_data));
+		return device;
+	}
+
+	ATADevice::ATADevice(BAN::RefPtr<ATABus> bus, ATABus::DeviceType type, bool is_secondary)
+		: m_bus(bus)
+		, m_type(type)
+		, m_is_secondary(is_secondary)
+	{ }
+
+	BAN::ErrorOr<void> ATADevice::read_sectors_impl(uint64_t lba, uint64_t sector_count, uint8_t* buffer)
+	{
+		TRY(m_bus->read(*this, lba, sector_count, buffer));
+		return {};
+	}
+
+	BAN::ErrorOr<void> ATADevice::write_sectors_impl(uint64_t lba, uint64_t sector_count, const uint8_t* buffer)
+	{
+		TRY(m_bus->write(*this, lba, sector_count, buffer));
+		return {};
+	}
+
 }

kernel/kernel/Storage/StorageDevice.cpp

@@ -283,9 +283,9 @@ namespace Kernel
 		return {};
 	}
 
-	BAN::ErrorOr<void> StorageDevice::read_sectors(uint64_t lba, uint8_t sector_count, uint8_t* buffer)
+	BAN::ErrorOr<void> StorageDevice::read_sectors(uint64_t lba, uint64_t sector_count, uint8_t* buffer)
 	{
-		for (uint8_t offset = 0; offset < sector_count; offset++)
+		for (uint64_t offset = 0; offset < sector_count; offset++)
 		{
 			LockGuard _(m_lock);
 			Thread::TerminateBlocker blocker(Thread::current());
@@ -302,7 +302,7 @@ namespace Kernel
 		return {};
 	}
 
-	BAN::ErrorOr<void> StorageDevice::write_sectors(uint64_t lba, uint8_t sector_count, const uint8_t* buffer)
+	BAN::ErrorOr<void> StorageDevice::write_sectors(uint64_t lba, uint64_t sector_count, const uint8_t* buffer)
 	{
 		// TODO: use disk cache for dirty pages. I don't wanna think about how to do it safely now
 		for (uint8_t offset = 0; offset < sector_count; offset++)

kernel/kernel/Timer/HPET.cpp

@@ -148,6 +148,12 @@ namespace Kernel
 		return read_register(HPET_REG_COUNTER) * m_counter_tick_period_fs / FS_PER_MS;
 	}
 
+	uint64_t HPET::ns_since_boot() const
+	{
+		// FIXME: 32 bit CPUs should use 32 bit counter with 32 bit reads
+		return read_register(HPET_REG_COUNTER) * m_counter_tick_period_fs / FS_PER_NS;
+	}
+
 	timespec HPET::time_since_boot() const
 	{
 		uint64_t time_fs = read_register(HPET_REG_COUNTER) * m_counter_tick_period_fs;

kernel/kernel/Timer/PIT.cpp

@@ -62,6 +62,11 @@ namespace Kernel
 		return m_system_time * (MS_PER_S / TICKS_PER_SECOND);
 	}
 
+	uint64_t PIT::ns_since_boot() const
+	{
+		return m_system_time * (NS_PER_S / TICKS_PER_SECOND);
+	}
+
 	timespec PIT::time_since_boot() const
 	{
 		uint64_t ticks = m_system_time;

kernel/kernel/Timer/Timer.cpp

@@ -59,6 +59,11 @@ namespace Kernel
 		return m_timer->ms_since_boot();
 	}
 
+	uint64_t SystemTimer::ns_since_boot() const
+	{
+		return m_timer->ns_since_boot();
+	}
+
 	timespec SystemTimer::time_since_boot() const
 	{
 		return m_timer->time_since_boot();

qemu.sh

@@ -1,8 +1,10 @@
 #!/bin/bash
 set -e
 
-qemu-system-$BANAN_ARCH										\
-	-m 128													\
-	-smp 2													\
-	-drive format=raw,media=disk,file=${DISK_IMAGE_PATH}	\
-	$@														\
+qemu-system-$BANAN_ARCH											\
+	-m 128														\
+	-smp 2														\
+	-drive format=raw,id=disk,file=${DISK_IMAGE_PATH},if=none	\
+	-device ahci,id=ahci										\
+	-device ide-hd,drive=disk,bus=ahci.0						\
+	$@															\