LibGUI/WindowServer: Rework packet serialization

Instead of sending while serializing (what even was that), we serialize
the whole packet into a buffer which can be sent in one go. First of all
this reduces the number of sends by a lot. This also fixes WindowServer
ending up sending partial packets when client is not responsive.
Previously we would just try sending once, if any send failed the send
was aborted while partial packet was already transmitted. This lead to
packet stream being out of sync leading to the client killing itself.
Now we allow 64 KiB outgoing buffer per client. If this buffer ever fills
up, we will not send partial packets.
This commit is contained in:
2026-04-07 09:13:34 +03:00
parent 2f9b8b6fc9
commit a4ba1da65a
7 changed files with 373 additions and 292 deletions

View File

@@ -16,7 +16,22 @@ Window::~Window()
smo_delete(m_smo_key);
LibGUI::EventPacket::DestroyWindowEvent packet;
(void)packet.send_serialized(m_client_fd);
BAN::Vector<uint8_t> buffer;
if (!buffer.resize(packet.serialized_size()).is_error())
{
packet.serialize(buffer.span());
size_t total_sent = 0;
while (total_sent < buffer.size())
{
const ssize_t nsend = send(m_client_fd, buffer.data() + total_sent, buffer.size() - total_sent, 0);
if (nsend <= 0)
break;
total_sent += nsend;
}
}
close(m_client_fd);
}

View File

@@ -86,7 +86,7 @@ void WindowServer::on_window_create(int fd, const LibGUI::WindowPacket::WindowCr
response.width = window->client_width();
response.height = window->client_height();
response.smo_key = window->smo_key();
if (auto ret = response.send_serialized(fd); ret.is_error())
if (auto ret = append_serialized_packet(response, fd); ret.is_error())
{
dwarnln("could not respond to window create request: {}", ret.error());
return;
@@ -205,7 +205,7 @@ void WindowServer::on_window_set_attributes(int fd, const LibGUI::WindowPacket::
.shown = target_window->get_attributes().shown,
},
};
if (auto ret = event_packet.send_serialized(target_window->client_fd()); ret.is_error())
if (auto ret = append_serialized_packet(event_packet, target_window->client_fd()); ret.is_error())
dwarnln("could not send window shown event: {}", ret.error());
if (packet.attributes.focusable && packet.attributes.shown && m_state == State::Normal)
@@ -312,7 +312,7 @@ void WindowServer::on_window_set_fullscreen(int fd, const LibGUI::WindowPacket::
auto event_packet = LibGUI::EventPacket::WindowFullscreenEvent {
.event = { .fullscreen = false }
};
if (auto ret = event_packet.send_serialized(m_focused_window->client_fd()); ret.is_error())
if (auto ret = append_serialized_packet(event_packet, m_focused_window->client_fd()); ret.is_error())
dwarnln("could not send window fullscreen event: {}", ret.error());
m_state = State::Normal;
@@ -347,7 +347,7 @@ void WindowServer::on_window_set_fullscreen(int fd, const LibGUI::WindowPacket::
auto event_packet = LibGUI::EventPacket::WindowFullscreenEvent {
.event = { .fullscreen = true }
};
if (auto ret = event_packet.send_serialized(target_window->client_fd()); ret.is_error())
if (auto ret = append_serialized_packet(event_packet, target_window->client_fd()); ret.is_error())
dwarnln("could not send window fullscreen event: {}", ret.error());
m_state = State::Fullscreen;
@@ -488,7 +488,7 @@ void WindowServer::on_key_event(LibInput::KeyEvent event)
if (m_is_mod_key_held && event.pressed() && event.key == LibInput::Key::Q)
{
LibGUI::EventPacket::CloseWindowEvent packet;
if (auto ret = packet.send_serialized(m_focused_window->client_fd()); ret.is_error())
if (auto ret = append_serialized_packet(packet, m_focused_window->client_fd()); ret.is_error())
dwarnln("could not send window close event: {}", ret.error());
return;
}
@@ -521,7 +521,7 @@ void WindowServer::on_key_event(LibInput::KeyEvent event)
auto event_packet = LibGUI::EventPacket::WindowFullscreenEvent {
.event = { .fullscreen = (m_state == State::Fullscreen) }
};
if (auto ret = event_packet.send_serialized(m_focused_window->client_fd()); ret.is_error())
if (auto ret = append_serialized_packet(event_packet, m_focused_window->client_fd()); ret.is_error())
dwarnln("could not send window fullscreen event: {}", ret.error());
invalidate(m_framebuffer.area());
@@ -530,7 +530,7 @@ void WindowServer::on_key_event(LibInput::KeyEvent event)
LibGUI::EventPacket::KeyEvent packet;
packet.event = event;
if (auto ret = packet.send_serialized(m_focused_window->client_fd()); ret.is_error())
if (auto ret = append_serialized_packet(packet, m_focused_window->client_fd()); ret.is_error())
dwarnln("could not send key event: {}", ret.error());
}
@@ -545,7 +545,7 @@ void WindowServer::on_mouse_button(LibInput::MouseButtonEvent event)
packet.event.pressed = event.pressed;
packet.event.x = 0;
packet.event.y = 0;
if (auto ret = packet.send_serialized(m_focused_window->client_fd()); ret.is_error())
if (auto ret = append_serialized_packet(packet, m_focused_window->client_fd()); ret.is_error())
dwarnln("could not send mouse button event: {}", ret.error());
return;
}
@@ -604,7 +604,7 @@ void WindowServer::on_mouse_button(LibInput::MouseButtonEvent event)
if (event.button == LibInput::MouseButton::Left && !event.pressed && target_window->close_button_area().contains(m_cursor))
{
LibGUI::EventPacket::CloseWindowEvent packet;
if (auto ret = packet.send_serialized(target_window->client_fd()); ret.is_error())
if (auto ret = append_serialized_packet(packet, target_window->client_fd()); ret.is_error())
dwarnln("could not send close window event: {}", ret.error());
break;
}
@@ -618,7 +618,7 @@ void WindowServer::on_mouse_button(LibInput::MouseButtonEvent event)
packet.event.pressed = event.pressed;
packet.event.x = m_cursor.x - target_window->client_x();
packet.event.y = m_cursor.y - target_window->client_y();
if (auto ret = packet.send_serialized(target_window->client_fd()); ret.is_error())
if (auto ret = append_serialized_packet(packet, target_window->client_fd()); ret.is_error())
{
dwarnln("could not send mouse button event: {}", ret.error());
return;
@@ -649,7 +649,7 @@ void WindowServer::on_mouse_button(LibInput::MouseButtonEvent event)
event.width = m_focused_window->client_width();
event.height = m_focused_window->client_height();
event.smo_key = m_focused_window->smo_key();
if (auto ret = event.send_serialized(m_focused_window->client_fd()); ret.is_error())
if (auto ret = append_serialized_packet(event, m_focused_window->client_fd()); ret.is_error())
{
dwarnln("could not respond to window resize request: {}", ret.error());
return;
@@ -698,7 +698,7 @@ void WindowServer::on_mouse_move_impl(int32_t new_x, int32_t new_y)
LibGUI::EventPacket::MouseMoveEvent packet;
packet.event.x = m_cursor.x - m_focused_window->client_x();
packet.event.y = m_cursor.y - m_focused_window->client_y();
if (auto ret = packet.send_serialized(m_focused_window->client_fd()); ret.is_error())
if (auto ret = append_serialized_packet(packet, m_focused_window->client_fd()); ret.is_error())
{
dwarnln("could not send mouse move event: {}", ret.error());
return;
@@ -736,7 +736,7 @@ void WindowServer::on_mouse_move(LibInput::MouseMoveEvent event)
LibGUI::EventPacket::MouseMoveEvent packet;
packet.event.x = event.rel_x;
packet.event.y = -event.rel_y;
if (auto ret = packet.send_serialized(m_focused_window->client_fd()); ret.is_error())
if (auto ret = append_serialized_packet(packet, m_focused_window->client_fd()); ret.is_error())
dwarnln("could not send mouse move event: {}", ret.error());
return;
}
@@ -807,7 +807,7 @@ void WindowServer::on_mouse_scroll(LibInput::MouseScrollEvent event)
{
LibGUI::EventPacket::MouseScrollEvent packet;
packet.event.scroll = event.scroll;
if (auto ret = packet.send_serialized(m_focused_window->client_fd()); ret.is_error())
if (auto ret = append_serialized_packet(packet, m_focused_window->client_fd()); ret.is_error())
{
dwarnln("could not send mouse scroll event: {}", ret.error());
return;
@@ -831,7 +831,7 @@ void WindowServer::set_focused_window(BAN::RefPtr<Window> window)
{
LibGUI::EventPacket::WindowFocusEvent packet;
packet.event.focused = false;
if (auto ret = packet.send_serialized(m_focused_window->client_fd()); ret.is_error())
if (auto ret = append_serialized_packet(packet, m_focused_window->client_fd()); ret.is_error())
dwarnln("could not send window focus event: {}", ret.error());
}
@@ -851,7 +851,7 @@ void WindowServer::set_focused_window(BAN::RefPtr<Window> window)
{
LibGUI::EventPacket::WindowFocusEvent packet;
packet.event.focused = true;
if (auto ret = packet.send_serialized(m_focused_window->client_fd()); ret.is_error())
if (auto ret = append_serialized_packet(packet, m_focused_window->client_fd()); ret.is_error())
dwarnln("could not send window focus event: {}", ret.error());
}
}
@@ -1526,7 +1526,7 @@ BAN::RefPtr<Window> WindowServer::find_hovered_window() const
return {};
}
bool WindowServer::resize_window(BAN::RefPtr<Window> window, uint32_t width, uint32_t height) const
bool WindowServer::resize_window(BAN::RefPtr<Window> window, uint32_t width, uint32_t height)
{
if (auto ret = window->resize(width, height); ret.is_error())
{
@@ -1538,7 +1538,7 @@ bool WindowServer::resize_window(BAN::RefPtr<Window> window, uint32_t width, uin
response.width = window->client_width();
response.height = window->client_height();
response.smo_key = window->smo_key();
if (auto ret = response.send_serialized(window->client_fd()); ret.is_error())
if (auto ret = append_serialized_packet(response, window->client_fd()); ret.is_error())
{
dwarnln("could not respond to window resize request: {}", ret.error());
return false;
@@ -1547,13 +1547,10 @@ bool WindowServer::resize_window(BAN::RefPtr<Window> window, uint32_t width, uin
return true;
}
void WindowServer::add_client_fd(int fd)
BAN::ErrorOr<void> WindowServer::add_client_fd(int fd)
{
if (auto ret = m_client_data.emplace(fd); ret.is_error())
{
dwarnln("could not add client: {}", ret.error());
return;
}
TRY(m_client_data.emplace(fd));
return {};
}
void WindowServer::remove_client_fd(int fd)
@@ -1612,3 +1609,31 @@ WindowServer::ClientData& WindowServer::get_client_data(int fd)
ASSERT_NOT_REACHED();
}
// TODO: this epoll stuff is very hacky
#include <sys/epoll.h>
extern int g_epoll_fd;
template<typename T>
BAN::ErrorOr<void> WindowServer::append_serialized_packet(const T& packet, int fd)
{
const size_t serialized_size = packet.serialized_size();
auto& client_data = m_client_data[fd];
if (client_data.out_buffer_size + serialized_size > client_data.out_buffer.size())
return BAN::Error::from_errno(ENOBUFS);
if (client_data.out_buffer_size == 0)
{
epoll_event event { .events = EPOLLIN | EPOLLOUT, .data = { .fd = fd } };
if (epoll_ctl(g_epoll_fd, EPOLL_CTL_MOD, fd, &event) == -1)
dwarnln("epoll_ctl add EPOLLOUT: {}", strerror(errno));
}
packet.serialize(client_data.out_buffer.span().slice(client_data.out_buffer_size, serialized_size));
client_data.out_buffer_size += serialized_size;
return {};
}

View File

@@ -20,8 +20,10 @@ class WindowServer
public:
struct ClientData
{
size_t packet_buffer_nread = 0;
BAN::Vector<uint8_t> packet_buffer;
size_t in_buffer_size { 0 };
BAN::Array<uint8_t, 64 * 1024> in_buffer;
size_t out_buffer_size { 0 };
BAN::Array<uint8_t, 64 * 1024> out_buffer;
};
public:
@@ -54,7 +56,7 @@ public:
Rectangle cursor_area() const;
Rectangle resize_area(Position cursor) const;
void add_client_fd(int fd);
BAN::ErrorOr<void> add_client_fd(int fd);
void remove_client_fd(int fd);
ClientData& get_client_data(int fd);
@@ -65,11 +67,14 @@ private:
void mark_pending_sync(Rectangle area);
bool resize_window(BAN::RefPtr<Window> window, uint32_t width, uint32_t height) const;
bool resize_window(BAN::RefPtr<Window> window, uint32_t width, uint32_t height);
BAN::RefPtr<Window> find_window_with_fd(int fd) const;
BAN::RefPtr<Window> find_hovered_window() const;
template<typename T>
BAN::ErrorOr<void> append_serialized_packet(const T& packet, int fd);
private:
struct RangeList
{

View File

@@ -145,6 +145,8 @@ int open_server_fd()
return server_fd;
}
int g_epoll_fd = -1;
int main()
{
srand(time(nullptr));
@@ -157,8 +159,8 @@ int main()
return 1;
}
int epoll_fd = epoll_create1(EPOLL_CLOEXEC);
if (epoll_fd == -1)
g_epoll_fd = epoll_create1(EPOLL_CLOEXEC);
if (g_epoll_fd == -1)
{
dwarnln("epoll_create1: {}", strerror(errno));
return 1;
@@ -169,7 +171,7 @@ int main()
.events = EPOLLIN,
.data = { .fd = server_fd },
};
if (epoll_ctl(epoll_fd, EPOLL_CTL_ADD, server_fd, &event) == -1)
if (epoll_ctl(g_epoll_fd, EPOLL_CTL_ADD, server_fd, &event) == -1)
{
dwarnln("epoll_ctl server: {}", strerror(errno));
return 1;
@@ -214,7 +216,7 @@ int main()
.events = EPOLLIN,
.data = { .fd = keyboard_fd },
};
if (epoll_ctl(epoll_fd, EPOLL_CTL_ADD, keyboard_fd, &event) == -1)
if (epoll_ctl(g_epoll_fd, EPOLL_CTL_ADD, keyboard_fd, &event) == -1)
{
dwarnln("epoll_ctl keyboard: {}", strerror(errno));
close(keyboard_fd);
@@ -231,7 +233,7 @@ int main()
.events = EPOLLIN,
.data = { .fd = mouse_fd },
};
if (epoll_ctl(epoll_fd, EPOLL_CTL_ADD, mouse_fd, &event) == -1)
if (epoll_ctl(g_epoll_fd, EPOLL_CTL_ADD, mouse_fd, &event) == -1)
{
dwarnln("epoll_ctl mouse: {}", strerror(errno));
close(mouse_fd);
@@ -283,7 +285,7 @@ int main()
timeout.tv_nsec = (sync_interval_us - (current_us - last_sync_us)) * 1000;
epoll_event events[16];
int epoll_events = epoll_pwait2(epoll_fd, events, 16, &timeout, nullptr);
int epoll_events = epoll_pwait2(g_epoll_fd, events, 16, &timeout, nullptr);
if (epoll_events == -1 && errno != EINTR)
{
dwarnln("epoll_pwait2: {}", strerror(errno));
@@ -296,25 +298,28 @@ int main()
{
ASSERT(events[i].events & EPOLLIN);
int window_fd = accept4(server_fd, nullptr, nullptr, SOCK_NONBLOCK | SOCK_CLOEXEC);
if (window_fd == -1)
int client_fd = accept4(server_fd, nullptr, nullptr, SOCK_NONBLOCK | SOCK_CLOEXEC);
if (client_fd == -1)
{
dwarnln("accept: {}", strerror(errno));
continue;
}
epoll_event event {
.events = EPOLLIN,
.data = { .fd = window_fd },
};
if (epoll_ctl(epoll_fd, EPOLL_CTL_ADD, window_fd, &event) == -1)
epoll_event event { .events = EPOLLIN, .data = { .fd = client_fd } };
if (epoll_ctl(g_epoll_fd, EPOLL_CTL_ADD, client_fd, &event) == -1)
{
dwarnln("epoll_ctl: {}", strerror(errno));
close(window_fd);
close(client_fd);
continue;
}
if (auto ret = window_server.add_client_fd(client_fd); ret.is_error())
{
dwarnln("add_client: {}", ret.error());
close(client_fd);
continue;
}
window_server.add_client_fd(window_fd);
continue;
}
@@ -361,99 +366,127 @@ int main()
}
const int client_fd = events[i].data.fd;
if (events[i].events & EPOLLHUP)
if (events[i].events & (EPOLLHUP | EPOLLERR))
{
epoll_ctl(epoll_fd, EPOLL_CTL_DEL, client_fd, nullptr);
epoll_ctl(g_epoll_fd, EPOLL_CTL_DEL, client_fd, nullptr);
window_server.remove_client_fd(client_fd);
continue;
}
ASSERT(events[i].events & EPOLLIN);
auto& client_data = window_server.get_client_data(client_fd);
if (client_data.packet_buffer.empty())
if (events[i].events & EPOLLOUT)
{
uint32_t packet_size;
const ssize_t nrecv = recv(client_fd, &packet_size, sizeof(uint32_t), 0);
if (nrecv < 0)
dwarnln("recv 1: {}", strerror(errno));
if (nrecv > 0 && nrecv != sizeof(uint32_t))
dwarnln("could not read packet size with a single recv call, closing connection...");
if (nrecv != sizeof(uint32_t))
ASSERT(client_data.out_buffer_size > 0);
const ssize_t nsend = send(client_fd, client_data.out_buffer.data(), client_data.out_buffer_size, 0);
if (nsend < 0 && !(errno == EWOULDBLOCK || errno == EAGAIN))
{
epoll_ctl(epoll_fd, EPOLL_CTL_DEL, client_fd, nullptr);
dwarnln("send: {}", strerror(errno));
epoll_ctl(g_epoll_fd, EPOLL_CTL_DEL, client_fd, nullptr);
window_server.remove_client_fd(client_fd);
break;
}
if (packet_size < 4)
if (nsend > 0)
{
dwarnln("client sent invalid packet, closing connection...");
epoll_ctl(epoll_fd, EPOLL_CTL_DEL, client_fd, nullptr);
window_server.remove_client_fd(client_fd);
break;
client_data.out_buffer_size -= nsend;
if (client_data.out_buffer_size == 0)
{
epoll_event event { .events = EPOLLIN, .data = { .fd = client_fd } };
if (epoll_ctl(g_epoll_fd, EPOLL_CTL_MOD, client_fd, &event) == -1)
dwarnln("epoll_ctl remove EPOLLOUT: {}", strerror(errno));
}
else
{
// TODO: maybe use a ring buffer so we don't have to memmove everything not sent
memmove(
client_data.out_buffer.data(),
client_data.out_buffer.data() + nsend,
client_data.out_buffer_size
);
}
}
// this is a bit harsh, but i don't want to work on skipping streaming packets
if (client_data.packet_buffer.resize(packet_size).is_error())
{
dwarnln("could not allocate memory for client packet, closing connection...");
epoll_ctl(epoll_fd, EPOLL_CTL_DEL, client_fd, nullptr);
window_server.remove_client_fd(client_fd);
break;
}
client_data.packet_buffer_nread = 0;
continue;
}
const ssize_t nrecv = recv(
client_fd,
client_data.packet_buffer.data() + client_data.packet_buffer_nread,
client_data.packet_buffer.size() - client_data.packet_buffer_nread,
0
);
if (nrecv < 0)
dwarnln("recv 2: {}", strerror(errno));
if (nrecv <= 0)
{
epoll_ctl(epoll_fd, EPOLL_CTL_DEL, client_fd, nullptr);
window_server.remove_client_fd(client_fd);
break;
}
client_data.packet_buffer_nread += nrecv;
if (client_data.packet_buffer_nread < client_data.packet_buffer.size())
if (!(events[i].events & EPOLLIN))
continue;
ASSERT(client_data.packet_buffer.size() >= sizeof(uint32_t));
switch (*reinterpret_cast<LibGUI::PacketType*>(client_data.packet_buffer.data()))
{
const ssize_t nrecv = recv(
client_fd,
client_data.in_buffer.data() + client_data.in_buffer_size,
client_data.in_buffer.size() - client_data.in_buffer_size,
0
);
if (nrecv < 0 && !(errno == EWOULDBLOCK || errno == EAGAIN))
{
dwarnln("recv: {}", strerror(errno));
epoll_ctl(g_epoll_fd, EPOLL_CTL_DEL, client_fd, nullptr);
window_server.remove_client_fd(client_fd);
break;
}
if (nrecv > 0)
client_data.in_buffer_size += nrecv;
}
size_t bytes_handled = 0;
while (client_data.in_buffer_size - bytes_handled >= sizeof(LibGUI::PacketHeader))
{
BAN::ConstByteSpan packet_span = client_data.in_buffer.span().slice(bytes_handled, client_data.in_buffer_size - bytes_handled);
const auto header = packet_span.as<const LibGUI::PacketHeader>();
if (packet_span.size() < header.size || header.size < sizeof(LibGUI::PacketHeader))
break;
packet_span = packet_span.slice(0, header.size);
switch (header.type)
{
#define WINDOW_PACKET_CASE(enum, function) \
case LibGUI::PacketType::enum: \
if (auto ret = LibGUI::WindowPacket::enum::deserialize(client_data.packet_buffer.span()); !ret.is_error()) \
window_server.function(client_fd, ret.release_value()); \
break
WINDOW_PACKET_CASE(WindowCreate, on_window_create);
WINDOW_PACKET_CASE(WindowInvalidate, on_window_invalidate);
WINDOW_PACKET_CASE(WindowSetPosition, on_window_set_position);
WINDOW_PACKET_CASE(WindowSetAttributes, on_window_set_attributes);
WINDOW_PACKET_CASE(WindowSetMouseRelative, on_window_set_mouse_relative);
WINDOW_PACKET_CASE(WindowSetSize, on_window_set_size);
WINDOW_PACKET_CASE(WindowSetMinSize, on_window_set_min_size);
WINDOW_PACKET_CASE(WindowSetMaxSize, on_window_set_max_size);
WINDOW_PACKET_CASE(WindowSetFullscreen, on_window_set_fullscreen);
WINDOW_PACKET_CASE(WindowSetTitle, on_window_set_title);
WINDOW_PACKET_CASE(WindowSetCursor, on_window_set_cursor);
case LibGUI::PacketType::enum: \
if (auto ret = LibGUI::WindowPacket::enum::deserialize(packet_span); !ret.is_error()) \
window_server.function(client_fd, ret.release_value()); \
else \
derrorln("invalid packet: {}", ret.error()); \
break
WINDOW_PACKET_CASE(WindowCreate, on_window_create);
WINDOW_PACKET_CASE(WindowInvalidate, on_window_invalidate);
WINDOW_PACKET_CASE(WindowSetPosition, on_window_set_position);
WINDOW_PACKET_CASE(WindowSetAttributes, on_window_set_attributes);
WINDOW_PACKET_CASE(WindowSetMouseRelative, on_window_set_mouse_relative);
WINDOW_PACKET_CASE(WindowSetSize, on_window_set_size);
WINDOW_PACKET_CASE(WindowSetMinSize, on_window_set_min_size);
WINDOW_PACKET_CASE(WindowSetMaxSize, on_window_set_max_size);
WINDOW_PACKET_CASE(WindowSetFullscreen, on_window_set_fullscreen);
WINDOW_PACKET_CASE(WindowSetTitle, on_window_set_title);
WINDOW_PACKET_CASE(WindowSetCursor, on_window_set_cursor);
#undef WINDOW_PACKET_CASE
default:
dprintln("unhandled packet type: {}", *reinterpret_cast<uint32_t*>(client_data.packet_buffer.data()));
default:
dprintln("unhandled packet type: {}", static_cast<uint32_t>(header.type));
break;
}
bytes_handled += header.size;
}
client_data.packet_buffer.clear();
client_data.packet_buffer_nread = 0;
// NOTE: this will only move a single partial packet, so this is fine
client_data.in_buffer_size -= bytes_handled;
memmove(
client_data.in_buffer.data(),
client_data.in_buffer.data() + bytes_handled,
client_data.in_buffer_size
);
if (client_data.in_buffer_size >= sizeof(LibGUI::PacketHeader))
{
const auto header = BAN::ConstByteSpan(client_data.in_buffer.span()).as<const LibGUI::PacketHeader>();
if (header.size < sizeof(LibGUI::PacketHeader) || header.size > client_data.in_buffer.size())
{
dwarnln("client tried to send a {} byte packet", header.size);
epoll_ctl(g_epoll_fd, EPOLL_CTL_DEL, client_fd, nullptr);
window_server.remove_client_fd(client_fd);
break;
}
}
}
}
}