LXR linux/tools/io_uring/queue.c

   1#include <sys/types.h>
   2#include <sys/stat.h>
   3#include <sys/mman.h>
   4#include <unistd.h>
   5#include <errno.h>
   6#include <string.h>
   7
   8#include "liburing.h"
   9#include "barrier.h"
  10
  11static int __io_uring_get_cqe(struct io_uring *ring,
  12                              struct io_uring_cqe **cqe_ptr, int wait)
  13{
  14        struct io_uring_cq *cq = &ring->cq;
  15        const unsigned mask = *cq->kring_mask;
  16        unsigned head;
  17        int ret;
  18
  19        *cqe_ptr = NULL;
  20        head = *cq->khead;
  21        do {
  22                /*
  23                 * It's necessary to use a read_barrier() before reading
  24                 * the CQ tail, since the kernel updates it locklessly. The
  25                 * kernel has the matching store barrier for the update. The
  26                 * kernel also ensures that previous stores to CQEs are ordered
  27                 * with the tail update.
  28                 */
  29                read_barrier();
  30                if (head != *cq->ktail) {
  31                        *cqe_ptr = &cq->cqes[head & mask];
  32                        break;
  33                }
  34                if (!wait)
  35                        break;
  36                ret = io_uring_enter(ring->ring_fd, 0, 1,
  37                                        IORING_ENTER_GETEVENTS, NULL);
  38                if (ret < 0)
  39                        return -errno;
  40        } while (1);
  41
  42        return 0;
  43}
  44
  45/*
  46 * Return an IO completion, if one is readily available. Returns 0 with
  47 * cqe_ptr filled in on success, -errno on failure.
  48 */
  49int io_uring_peek_cqe(struct io_uring *ring, struct io_uring_cqe **cqe_ptr)
  50{
  51        return __io_uring_get_cqe(ring, cqe_ptr, 0);
  52}
  53
  54/*
  55 * Return an IO completion, waiting for it if necessary. Returns 0 with
  56 * cqe_ptr filled in on success, -errno on failure.
  57 */
  58int io_uring_wait_cqe(struct io_uring *ring, struct io_uring_cqe **cqe_ptr)
  59{
  60        return __io_uring_get_cqe(ring, cqe_ptr, 1);
  61}
  62
  63/*
  64 * Submit sqes acquired from io_uring_get_sqe() to the kernel.
  65 *
  66 * Returns number of sqes submitted
  67 */
  68int io_uring_submit(struct io_uring *ring)
  69{
  70        struct io_uring_sq *sq = &ring->sq;
  71        const unsigned mask = *sq->kring_mask;
  72        unsigned ktail, ktail_next, submitted, to_submit;
  73        int ret;
  74
  75        /*
  76         * If we have pending IO in the kring, submit it first. We need a
  77         * read barrier here to match the kernels store barrier when updating
  78         * the SQ head.
  79         */
  80        read_barrier();
  81        if (*sq->khead != *sq->ktail) {
  82                submitted = *sq->kring_entries;
  83                goto submit;
  84        }
  85
  86        if (sq->sqe_head == sq->sqe_tail)
  87                return 0;
  88
  89        /*
  90         * Fill in sqes that we have queued up, adding them to the kernel ring
  91         */
  92        submitted = 0;
  93        ktail = ktail_next = *sq->ktail;
  94        to_submit = sq->sqe_tail - sq->sqe_head;
  95        while (to_submit--) {
  96                ktail_next++;
  97                read_barrier();
  98
  99                sq->array[ktail & mask] = sq->sqe_head & mask;
 100                ktail = ktail_next;
 101
 102                sq->sqe_head++;
 103                submitted++;
 104        }
 105
 106        if (!submitted)
 107                return 0;
 108
 109        if (*sq->ktail != ktail) {
 110                /*
 111                 * First write barrier ensures that the SQE stores are updated
 112                 * with the tail update. This is needed so that the kernel
 113                 * will never see a tail update without the preceeding sQE
 114                 * stores being done.
 115                 */
 116                write_barrier();
 117                *sq->ktail = ktail;
 118                /*
 119                 * The kernel has the matching read barrier for reading the
 120                 * SQ tail.
 121                 */
 122                write_barrier();
 123        }
 124
 125submit:
 126        ret = io_uring_enter(ring->ring_fd, submitted, 0,
 127                                IORING_ENTER_GETEVENTS, NULL);
 128        if (ret < 0)
 129                return -errno;
 130
 131        return ret;
 132}
 133
 134/*
 135 * Return an sqe to fill. Application must later call io_uring_submit()
 136 * when it's ready to tell the kernel about it. The caller may call this
 137 * function multiple times before calling io_uring_submit().
 138 *
 139 * Returns a vacant sqe, or NULL if we're full.
 140 */
 141struct io_uring_sqe *io_uring_get_sqe(struct io_uring *ring)
 142{
 143        struct io_uring_sq *sq = &ring->sq;
 144        unsigned next = sq->sqe_tail + 1;
 145        struct io_uring_sqe *sqe;
 146
 147        /*
 148         * All sqes are used
 149         */
 150        if (next - sq->sqe_head > *sq->kring_entries)
 151                return NULL;
 152
 153        sqe = &sq->sqes[sq->sqe_tail & *sq->kring_mask];
 154        sq->sqe_tail = next;
 155        return sqe;
 156}
 157