tools/io_uring/queue.c

   1 #include <sys/types.h>
   2 #include <sys/stat.h>
   3 #include <sys/mman.h>
   4 #include <unistd.h>
   5 #include <errno.h>
   6 #include <string.h>
   7
   8 #include "liburing.h"
   9 #include "barrier.h"
  10
  11 static int __io_uring_get_cqe(struct io_uring *ring,
  12                               struct io_uring_cqe **cqe_ptr, int wait)
  13 {
  14         struct io_uring_cq *cq = &ring->cq;
  15         const unsigned mask = *cq->kring_mask;
  16         unsigned head;
  17         int ret;
  18
  19         *cqe_ptr = NULL;
  20         head = *cq->khead;
  21         do {
  22                 /*
  23                  * It's necessary to use a read_barrier() before reading
  24                  * the CQ tail, since the kernel updates it locklessly. The
  25                  * kernel has the matching store barrier for the update. The
  26                  * kernel also ensures that previous stores to CQEs are ordered
  27                  * with the tail update.
  28                  */
  29                 read_barrier();
  30                 if (head != *cq->ktail) {
  31                         *cqe_ptr = &cq->cqes[head & mask];
  32                         break;
  33                 }
  34                 if (!wait)
  35                         break;
  36                 ret = io_uring_enter(ring->ring_fd, 0, 1,
  37                                         IORING_ENTER_GETEVENTS, NULL);
  38                 if (ret < 0)
  39                         return -errno;
  40         } while (1);
  41
  42         return 0;
  43 }
  44
  45 /*
  46  * Return an IO completion, if one is readily available. Returns 0 with
  47  * cqe_ptr filled in on success, -errno on failure.
  48  */
  49 int io_uring_peek_cqe(struct io_uring *ring, struct io_uring_cqe **cqe_ptr)
  50 {
  51         return __io_uring_get_cqe(ring, cqe_ptr, 0);
  52 }
  53
  54 /*
  55  * Return an IO completion, waiting for it if necessary. Returns 0 with
  56  * cqe_ptr filled in on success, -errno on failure.
  57  */
  58 int io_uring_wait_cqe(struct io_uring *ring, struct io_uring_cqe **cqe_ptr)
  59 {
  60         return __io_uring_get_cqe(ring, cqe_ptr, 1);
  61 }
  62
  63 /*
  64  * Submit sqes acquired from io_uring_get_sqe() to the kernel.
  65  *
  66  * Returns number of sqes submitted
  67  */
  68 int io_uring_submit(struct io_uring *ring)
  69 {
  70         struct io_uring_sq *sq = &ring->sq;
  71         const unsigned mask = *sq->kring_mask;
  72         unsigned ktail, ktail_next, submitted, to_submit;
  73         int ret;
  74
  75         /*
  76          * If we have pending IO in the kring, submit it first. We need a
  77          * read barrier here to match the kernels store barrier when updating
  78          * the SQ head.
  79          */
  80         read_barrier();
  81         if (*sq->khead != *sq->ktail) {
  82                 submitted = *sq->kring_entries;
  83                 goto submit;
  84         }
  85
  86         if (sq->sqe_head == sq->sqe_tail)
  87                 return 0;
  88
  89         /*
  90          * Fill in sqes that we have queued up, adding them to the kernel ring
  91          */
  92         submitted = 0;
  93         ktail = ktail_next = *sq->ktail;
  94         to_submit = sq->sqe_tail - sq->sqe_head;
  95         while (to_submit--) {
  96                 ktail_next++;
  97                 read_barrier();
  98
  99                 sq->array[ktail & mask] = sq->sqe_head & mask;
 100                 ktail = ktail_next;
 101
 102                 sq->sqe_head++;
 103                 submitted++;
 104         }
 105
 106         if (!submitted)
 107                 return 0;
 108
 109         if (*sq->ktail != ktail) {
 110                 /*
 111                  * First write barrier ensures that the SQE stores are updated
 112                  * with the tail update. This is needed so that the kernel
 113                  * will never see a tail update without the preceeding sQE
 114                  * stores being done.
 115                  */
 116                 write_barrier();
 117                 *sq->ktail = ktail;
 118                 /*
 119                  * The kernel has the matching read barrier for reading the
 120                  * SQ tail.
 121                  */
 122                 write_barrier();
 123         }
 124
 125 submit:
 126         ret = io_uring_enter(ring->ring_fd, submitted, 0,
 127                                 IORING_ENTER_GETEVENTS, NULL);
 128         if (ret < 0)
 129                 return -errno;
 130
 131         return ret;
 132 }
 133
 134 /*
 135  * Return an sqe to fill. Application must later call io_uring_submit()
 136  * when it's ready to tell the kernel about it. The caller may call this
 137  * function multiple times before calling io_uring_submit().
 138  *
 139  * Returns a vacant sqe, or NULL if we're full.
 140  */
 141 struct io_uring_sqe *io_uring_get_sqe(struct io_uring *ring)
 142 {
 143         struct io_uring_sq *sq = &ring->sq;
 144         unsigned next = sq->sqe_tail + 1;
 145         struct io_uring_sqe *sqe;
 146
 147         /*
 148          * All sqes are used
 149          */
 150         if (next - sq->sqe_head > *sq->kring_entries)
 151                 return NULL;
 152
 153         sqe = &sq->sqes[sq->sqe_tail & *sq->kring_mask];
 154         sq->sqe_tail = next;
 155         return sqe;
 156 }