LXR linux/drivers/infiniband/sw/rxe/rxe

   1/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
   2/*
   3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
   4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
   5 */
   6
   7#ifndef RXE_QUEUE_H
   8#define RXE_QUEUE_H
   9
  10/* for definition of shared struct rxe_queue_buf */
  11#include <uapi/rdma/rdma_user_rxe.h>
  12
  13/* Implements a simple circular buffer that is shared between user
  14 * and the driver and can be resized. The requested element size is
  15 * rounded up to a power of 2 and the number of elements in the buffer
  16 * is also rounded up to a power of 2. Since the queue is empty when
  17 * the producer and consumer indices match the maximum capacity of the
  18 * queue is one less than the number of element slots.
  19 *
  20 * Notes:
  21 *   - The driver indices are always masked off to q->index_mask
  22 *     before storing so do not need to be checked on reads.
  23 *   - The user whether user space or kernel is generally
  24 *     not trusted so its parameters are masked to make sure
  25 *     they do not access the queue out of bounds on reads.
  26 *   - The driver indices for queues must not be written
  27 *     by user so a local copy is used and a shared copy is
  28 *     stored when the local copy is changed.
  29 *   - By passing the type in the parameter list separate from q
  30 *     the compiler can eliminate the switch statement when the
  31 *     actual queue type is known when the function is called at
  32 *     compile time.
  33 *   - These queues are lock free. The user and driver must protect
  34 *     changes to their end of the queues with locks if more than one
  35 *     CPU can be accessing it at the same time.
  36 */
  37
  38/**
  39 * enum queue_type - type of queue
  40 * @QUEUE_TYPE_TO_CLIENT:       Queue is written by rxe driver and
  41 *                              read by client. Used by rxe driver only.
  42 * @QUEUE_TYPE_FROM_CLIENT:     Queue is written by client and
  43 *                              read by rxe driver. Used by rxe driver only.
  44 * @QUEUE_TYPE_TO_DRIVER:       Queue is written by client and
  45 *                              read by rxe driver. Used by kernel client only.
  46 * @QUEUE_TYPE_FROM_DRIVER:     Queue is written by rxe driver and
  47 *                              read by client. Used by kernel client only.
  48 */
  49enum queue_type {
  50        QUEUE_TYPE_TO_CLIENT,
  51        QUEUE_TYPE_FROM_CLIENT,
  52        QUEUE_TYPE_TO_DRIVER,
  53        QUEUE_TYPE_FROM_DRIVER,
  54};
  55
  56struct rxe_queue {
  57        struct rxe_dev          *rxe;
  58        struct rxe_queue_buf    *buf;
  59        struct rxe_mmap_info    *ip;
  60        size_t                  buf_size;
  61        size_t                  elem_size;
  62        unsigned int            log2_elem_size;
  63        u32                     index_mask;
  64        enum queue_type         type;
  65        /* private copy of index for shared queues between
  66         * kernel space and user space. Kernel reads and writes
  67         * this copy and then replicates to rxe_queue_buf
  68         * for read access by user space.
  69         */
  70        u32                     index;
  71};
  72
  73int do_mmap_info(struct rxe_dev *rxe, struct mminfo __user *outbuf,
  74                 struct ib_udata *udata, struct rxe_queue_buf *buf,
  75                 size_t buf_size, struct rxe_mmap_info **ip_p);
  76
  77void rxe_queue_reset(struct rxe_queue *q);
  78
  79struct rxe_queue *rxe_queue_init(struct rxe_dev *rxe, int *num_elem,
  80                        unsigned int elem_size, enum queue_type type);
  81
  82int rxe_queue_resize(struct rxe_queue *q, unsigned int *num_elem_p,
  83                     unsigned int elem_size, struct ib_udata *udata,
  84                     struct mminfo __user *outbuf,
  85                     spinlock_t *producer_lock, spinlock_t *consumer_lock);
  86
  87void rxe_queue_cleanup(struct rxe_queue *queue);
  88
  89static inline u32 queue_next_index(struct rxe_queue *q, int index)
  90{
  91        return (index + 1) & q->index_mask;
  92}
  93
  94static inline u32 queue_get_producer(const struct rxe_queue *q,
  95                                     enum queue_type type)
  96{
  97        u32 prod;
  98
  99        switch (type) {
 100        case QUEUE_TYPE_FROM_CLIENT:
 101                /* protect user index */
 102                prod = smp_load_acquire(&q->buf->producer_index);
 103                break;
 104        case QUEUE_TYPE_TO_CLIENT:
 105                prod = q->index;
 106                break;
 107        case QUEUE_TYPE_FROM_DRIVER:
 108                /* protect driver index */
 109                prod = smp_load_acquire(&q->buf->producer_index);
 110                break;
 111        case QUEUE_TYPE_TO_DRIVER:
 112                prod = q->buf->producer_index;
 113                break;
 114        }
 115
 116        return prod;
 117}
 118
 119static inline u32 queue_get_consumer(const struct rxe_queue *q,
 120                                     enum queue_type type)
 121{
 122        u32 cons;
 123
 124        switch (type) {
 125        case QUEUE_TYPE_FROM_CLIENT:
 126                cons = q->index;
 127                break;
 128        case QUEUE_TYPE_TO_CLIENT:
 129                /* protect user index */
 130                cons = smp_load_acquire(&q->buf->consumer_index);
 131                break;
 132        case QUEUE_TYPE_FROM_DRIVER:
 133                cons = q->buf->consumer_index;
 134                break;
 135        case QUEUE_TYPE_TO_DRIVER:
 136                /* protect driver index */
 137                cons = smp_load_acquire(&q->buf->consumer_index);
 138                break;
 139        }
 140
 141        return cons;
 142}
 143
 144static inline int queue_empty(struct rxe_queue *q, enum queue_type type)
 145{
 146        u32 prod = queue_get_producer(q, type);
 147        u32 cons = queue_get_consumer(q, type);
 148
 149        return ((prod - cons) & q->index_mask) == 0;
 150}
 151
 152static inline int queue_full(struct rxe_queue *q, enum queue_type type)
 153{
 154        u32 prod = queue_get_producer(q, type);
 155        u32 cons = queue_get_consumer(q, type);
 156
 157        return ((prod + 1 - cons) & q->index_mask) == 0;
 158}
 159
 160static inline u32 queue_count(const struct rxe_queue *q,
 161                                        enum queue_type type)
 162{
 163        u32 prod = queue_get_producer(q, type);
 164        u32 cons = queue_get_consumer(q, type);
 165
 166        return (prod - cons) & q->index_mask;
 167}
 168
 169static inline void queue_advance_producer(struct rxe_queue *q,
 170                                          enum queue_type type)
 171{
 172        u32 prod;
 173
 174        switch (type) {
 175        case QUEUE_TYPE_FROM_CLIENT:
 176                pr_warn("%s: attempt to advance client index\n",
 177                        __func__);
 178                break;
 179        case QUEUE_TYPE_TO_CLIENT:
 180                prod = q->index;
 181                prod = (prod + 1) & q->index_mask;
 182                q->index = prod;
 183                /* protect user index */
 184                smp_store_release(&q->buf->producer_index, prod);
 185                break;
 186        case QUEUE_TYPE_FROM_DRIVER:
 187                pr_warn("%s: attempt to advance driver index\n",
 188                        __func__);
 189                break;
 190        case QUEUE_TYPE_TO_DRIVER:
 191                prod = q->buf->producer_index;
 192                prod = (prod + 1) & q->index_mask;
 193                q->buf->producer_index = prod;
 194                break;
 195        }
 196}
 197
 198static inline void queue_advance_consumer(struct rxe_queue *q,
 199                                          enum queue_type type)
 200{
 201        u32 cons;
 202
 203        switch (type) {
 204        case QUEUE_TYPE_FROM_CLIENT:
 205                cons = q->index;
 206                cons = (cons + 1) & q->index_mask;
 207                q->index = cons;
 208                /* protect user index */
 209                smp_store_release(&q->buf->consumer_index, cons);
 210                break;
 211        case QUEUE_TYPE_TO_CLIENT:
 212                pr_warn("%s: attempt to advance client index\n",
 213                        __func__);
 214                break;
 215        case QUEUE_TYPE_FROM_DRIVER:
 216                cons = q->buf->consumer_index;
 217                cons = (cons + 1) & q->index_mask;
 218                q->buf->consumer_index = cons;
 219                break;
 220        case QUEUE_TYPE_TO_DRIVER:
 221                pr_warn("%s: attempt to advance driver index\n",
 222                        __func__);
 223                break;
 224        }
 225}
 226
 227static inline void *queue_producer_addr(struct rxe_queue *q,
 228                                        enum queue_type type)
 229{
 230        u32 prod = queue_get_producer(q, type);
 231
 232        return q->buf->data + (prod << q->log2_elem_size);
 233}
 234
 235static inline void *queue_consumer_addr(struct rxe_queue *q,
 236                                        enum queue_type type)
 237{
 238        u32 cons = queue_get_consumer(q, type);
 239
 240        return q->buf->data + (cons << q->log2_elem_size);
 241}
 242
 243static inline void *queue_addr_from_index(struct rxe_queue *q, u32 index)
 244{
 245        return q->buf->data + ((index & q->index_mask)
 246                                << q->log2_elem_size);
 247}
 248
 249static inline u32 queue_index_from_addr(const struct rxe_queue *q,
 250                                const void *addr)
 251{
 252        return (((u8 *)addr - q->buf->data) >> q->log2_elem_size)
 253                                & q->index_mask;
 254}
 255
 256static inline void *queue_head(struct rxe_queue *q, enum queue_type type)
 257{
 258        return queue_empty(q, type) ? NULL : queue_consumer_addr(q, type);
 259}
 260
 261#endif /* RXE_QUEUE_H */
 262