view release on metacpan or  search on metacpan

lib/Hypersonic/Event/IOUring.pm  view on Meta::CPAN

package Hypersonic::Event::IOUring;

use strict;
use warnings;
use 5.010;

use parent 'Hypersonic::Event::Role';

our $VERSION = '0.19';

sub name { 'io_uring' }

sub available {
    return 0 unless $^O eq 'linux';

    # Check kernel version >= 5.13.
    #
    # We need kernel 5.13+ (not just 5.1+) because the readiness-only
    # backend in 0.19+ uses io_uring_prep_poll_multishot() which was
    # added in Linux 5.13 / liburing 2.1 (Aug 2021). Multi-shot poll
    # is essential: with one-shot poll_add the userspace re-arm in
    # gen_get_fd races against the main loop's recv() in a way that
    # makes the freshly re-armed (level-triggered) poll fire while
    # the buffer still has unread data, then fire AGAIN with an
    # empty buffer after recv() drains, causing the next iteration
    # to recv() and get EAGAIN, which the main loop treats as a
    # disconnect. This bug killed all sequential-keep-alive tests
    # (t/2100..t/2102, t/0035 WebSocket echo) the first time we
    # tried readiness-only mode. Multi-shot lets the kernel manage
    # the re-arm atomically with the readiness check, avoiding the
    # race entirely.
    #
    # Kernels < 5.13 fall back to epoll automatically via
    # Hypersonic::Event::best_backend's priority list. cpansmoker
    # hosts on Debian 12 (kernel 6.1+) and Fedora 43 (kernel 6.x)
    # all satisfy this; Debian 11 (5.10) and older fall back.
    my $ver = `uname -r 2>/dev/null` || '';
    my ($major, $minor) = $ver =~ /^(\d+)\.(\d+)/;
    return 0 unless $major && ($major > 5 || ($major == 5 && $minor >= 13));

    # Check for liburing headers
    my $has_header = -f '/usr/include/liburing.h'
        || -f '/usr/local/include/liburing.h'
        || -f '/usr/include/x86_64-linux-gnu/liburing.h';
    return 0 unless $has_header;

    # io_uring may be disabled at the kernel level. RHEL9 ships with
    # kernel.io_uring_disabled=2 by default; a value of 1 or 2 means
    # the syscall returns EINVAL/EPERM regardless of liburing being
    # linkable. Bail before we sink time into a compile+link probe.
    if (open my $fh, '<', '/proc/sys/kernel/io_uring_disabled') {
        my $disabled = <$fh>;
        close $fh;
        chomp $disabled if defined $disabled;
        return 0 if defined $disabled && $disabled ne '0';
    }

    # Compile-link-and-RUN probe. A pure link check passes on systems
    # that have liburing installed but where io_uring_setup() will
    # nevertheless fail at runtime (kernel disabled, sandboxing, missing
    # liburing.so at exec time). Also probe for io_uring_prep_poll_multishot
    # symbol availability - the symbol was added in liburing 2.1, and
    # the actual multishot poll operation requires kernel 5.13+. If
    # either is missing we want to fall back to epoll silently.
    require Hypersonic::JIT::Util;
    return Hypersonic::JIT::Util->can_run(
        '',
        '-luring',
        'struct io_uring ring; int rc = io_uring_queue_init(8, &ring, 0); '
            . 'if (rc < 0) return 1; '
            . 'struct io_uring_sqe* sqe = io_uring_get_sqe(&ring); '
            . 'if (!sqe) { io_uring_queue_exit(&ring); return 2; } '
            . 'io_uring_prep_poll_multishot(sqe, 0, POLLIN); '
            . 'io_uring_sqe_set_data(sqe, (void*)0); '
            . 'io_uring_queue_exit(&ring); return 0;',
        "#include <liburing.h>\n#include <poll.h>",
    );
}

sub includes {
    # liburing.h for the server loop.
    # <poll.h> for POLLIN (the readiness mask we pass to prep_poll_add).
    # <sys/epoll.h> is needed for the UA::Async slot-tracking helpers
    # (gen_create_loop / _add_with_slot / _get_slot) - io_uring is
    # Linux 5.1+ which always has epoll.
    return "#include <liburing.h>\n#include <poll.h>\n#include <sys/epoll.h>";
}

lib/Hypersonic/Event/IOUring.pm  view on Meta::CPAN

# table; the underlying socket stays open (and the peer never sees a
# TCP FIN) until io_uring drops its file reference. Since
# io_uring_prep_cancel is async, that drop happens at an unspecified
# later time -- the client can wait indefinitely for the EOF that
# tells it "the response is complete".
#
# Symptom: short-lived `Connection: close` HTTP requests appear to
# succeed on the server (response is fully sent) but the client's
# blocking recv() loop never returns 0. Tests like t/2100 hang at
# the first POST.
#
# Fix: call shutdown(fd, SHUT_RDWR) here BEFORE submitting the
# cancel. shutdown operates on the socket directly and unconditionally
# sends FIN to the peer regardless of any reference counts.
# io_uring's struct file ref is irrelevant to whether TCP FIN goes
# out. The caller's close() afterwards still does the right thing
# (marks the fd unused in our process); io_uring will eventually
# release its own ref when the cancel completes async.
#
# Bumping the generation counter (still done) is what closes the
# fd-reuse race: from this moment on, any pending CQE that still
# carries the old generation is silently discarded by gen_get_fd,
# even if accept() reuses the fd number before the cancel completes.
#
# We use io_uring_prep_cancel (not io_uring_prep_poll_remove) because
# poll_remove's signature flipped from void* to __u64 around liburing
# 2.0 whereas prep_cancel's void* user_data is stable from 0.7+.
#
# The cancel SQE carries user_data=0 so its own CQE is cheaply
# distinguishable from real poll CQEs (which always have non-zero
# user_data thanks to the generation in the high 32 bits).
sub gen_del {
    my ($class, $builder, $loop_var, $fd_var) = @_;

    $builder->line('{')
      ->line("    if ($fd_var >= 0 && $fd_var < MAX_FD) {")
      ->line("        g_iouring_fd_gen[$fd_var]++;")
      ->line('    }')
      ->comment('Force-send TCP FIN regardless of io_uring file refs')
      ->line("    shutdown($fd_var, SHUT_RDWR);")
      ->line('    struct io_uring_sqe* _dsqe = io_uring_get_sqe(&ring);')
      ->line('    if (_dsqe) {')
      ->line("        io_uring_prep_cancel(_dsqe, (void*)(uintptr_t)$fd_var, 0);")
      ->line('        io_uring_sqe_set_data(_dsqe, NULL);')
      ->line('        io_uring_submit(&ring);')
      ->line('    }')
      ->line('}');
}

# Copy CQEs out of the ring buffer into our private value array, then
# release all consumed ring slots at once with io_uring_cq_advance.
# This avoids BUG 1 (pointer staleness) - we never reference ring
# slots after they've been released. We also lose nothing functionally
# because the only fields we ever need from a CQE are user_data and
# res.
#
# CRUCIAL: do NOT `continue;` on -ETIME or -EINTR. The main loop's
# shutdown-drain branch (which force-closes all connections when
# g_shutdown is set) lives AFTER gen_wait but BEFORE the event-
# processing loop. If we `continue;` here, we never reach that branch,
# and a server with idle keep-alive connections that gets SIGTERMed
# will spin in gen_wait forever (no CQEs arriving means perpetual
# -ETIME). Instead, set count=0 and fall through so the shutdown
# branch runs and the cleanup pass drains the connections. This is
# the same shape as epoll_wait()=0 in the Epoll backend.
sub gen_wait {
    my ($class, $builder, $loop_var, $events_var, $count_var, $timeout_var) = @_;

    $builder->line('struct io_uring_cqe* cqe;')
      ->line('struct __kernel_timespec ts;')
      ->line("ts.tv_sec = $timeout_var / 1000;")
      ->line("ts.tv_nsec = ($timeout_var % 1000) * 1000000;")
      ->blank
      ->line("int $count_var = 0;")
      ->line('static hs_iouring_event_t events_buf[MAX_EVENTS];')
      ->line("$events_var = events_buf;")
      ->blank
      ->comment('Block until at least one completion is ready')
      ->line('int wait_result = io_uring_wait_cqe_timeout(&ring, &cqe, &ts);')
      ->if('wait_result == 0')
        ->comment('Drain all currently available CQEs (BUG 1 fix: copy values)')
        ->line('unsigned head;')
        ->line('io_uring_for_each_cqe(&ring, head, cqe) {')
        ->line("    if ($count_var < MAX_EVENTS) {")
        ->line("        events_buf[$count_var].ud  = (uint64_t)(uintptr_t)io_uring_cqe_get_data(cqe);")
        ->line("        events_buf[$count_var].res = cqe->res;")
        ->line("        $count_var++;")
        ->line('    }')
        ->line('}')
        ->line("io_uring_cq_advance(&ring, (unsigned)$count_var);")
      ->elsif('wait_result == -ETIME || wait_result == -EINTR')
        ->comment('Timeout / signal: fall through with count=0 so the')
        ->comment('cleanup-on-shutdown branch can run. Do NOT continue;')
      ->else
        ->line('break;')
      ->endif;
}

# Extract fd from our private value-array CQE. NO io_uring_cqe_seen
# call here - gen_wait already advanced the ring cursor once for the
# whole batch.
#
# Filters applied (any failure -> continue, skip this event):
#   * ud == 0           -> CQE is from a cancel SQE
#   * res < 0           -> poll cancelled/errored (-ECANCELED, -EBADF, ...)
#   * fd out of range   -> defensive guard against corruption
#   * stale generation  -> fd was closed and reused, this CQE is for the
#                          old lifetime (see BUG 2 in defines() comment)
sub gen_get_fd {
    my ($class, $builder, $events_var, $index_var, $fd_var) = @_;

    $builder->line("uint64_t _ud  = ${events_var}[$index_var].ud;")
      ->line("int      _res = ${events_var}[$index_var].res;")
      ->if('_ud == 0')
        ->line('continue;')  # cancel-SQE completion
      ->endif
      ->if('_res < 0')
        ->line('continue;')  # poll cancelled / errored
      ->endif
      ->line('uint32_t _ud_gen = (uint32_t)(_ud >> 32);')
      ->line("int $fd_var    = (int)(_ud & 0xFFFFFFFFu);")