LXR linux/drivers/infiniband/sw/rdmavt/rc.c

   1// SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
   2/*
   3 * Copyright(c) 2016 Intel Corporation.
   4 */
   5
   6#include <rdma/rdmavt_qp.h>
   7#include <rdma/ib_hdrs.h>
   8
   9/*
  10 * Convert the AETH credit code into the number of credits.
  11 */
  12static const u16 credit_table[31] = {
  13        0,                      /* 0 */
  14        1,                      /* 1 */
  15        2,                      /* 2 */
  16        3,                      /* 3 */
  17        4,                      /* 4 */
  18        6,                      /* 5 */
  19        8,                      /* 6 */
  20        12,                     /* 7 */
  21        16,                     /* 8 */
  22        24,                     /* 9 */
  23        32,                     /* A */
  24        48,                     /* B */
  25        64,                     /* C */
  26        96,                     /* D */
  27        128,                    /* E */
  28        192,                    /* F */
  29        256,                    /* 10 */
  30        384,                    /* 11 */
  31        512,                    /* 12 */
  32        768,                    /* 13 */
  33        1024,                   /* 14 */
  34        1536,                   /* 15 */
  35        2048,                   /* 16 */
  36        3072,                   /* 17 */
  37        4096,                   /* 18 */
  38        6144,                   /* 19 */
  39        8192,                   /* 1A */
  40        12288,                  /* 1B */
  41        16384,                  /* 1C */
  42        24576,                  /* 1D */
  43        32768                   /* 1E */
  44};
  45
  46/**
  47 * rvt_compute_aeth - compute the AETH (syndrome + MSN)
  48 * @qp: the queue pair to compute the AETH for
  49 *
  50 * Returns the AETH.
  51 */
  52__be32 rvt_compute_aeth(struct rvt_qp *qp)
  53{
  54        u32 aeth = qp->r_msn & IB_MSN_MASK;
  55
  56        if (qp->ibqp.srq) {
  57                /*
  58                 * Shared receive queues don't generate credits.
  59                 * Set the credit field to the invalid value.
  60                 */
  61                aeth |= IB_AETH_CREDIT_INVAL << IB_AETH_CREDIT_SHIFT;
  62        } else {
  63                u32 min, max, x;
  64                u32 credits;
  65                u32 head;
  66                u32 tail;
  67
  68                credits = READ_ONCE(qp->r_rq.kwq->count);
  69                if (credits == 0) {
  70                        /* sanity check pointers before trusting them */
  71                        if (qp->ip) {
  72                                head = RDMA_READ_UAPI_ATOMIC(qp->r_rq.wq->head);
  73                                tail = RDMA_READ_UAPI_ATOMIC(qp->r_rq.wq->tail);
  74                        } else {
  75                                head = READ_ONCE(qp->r_rq.kwq->head);
  76                                tail = READ_ONCE(qp->r_rq.kwq->tail);
  77                        }
  78                        if (head >= qp->r_rq.size)
  79                                head = 0;
  80                        if (tail >= qp->r_rq.size)
  81                                tail = 0;
  82                        /*
  83                         * Compute the number of credits available (RWQEs).
  84                         * There is a small chance that the pair of reads are
  85                         * not atomic, which is OK, since the fuzziness is
  86                         * resolved as further ACKs go out.
  87                         */
  88                        credits = rvt_get_rq_count(&qp->r_rq, head, tail);
  89                }
  90                /*
  91                 * Binary search the credit table to find the code to
  92                 * use.
  93                 */
  94                min = 0;
  95                max = 31;
  96                for (;;) {
  97                        x = (min + max) / 2;
  98                        if (credit_table[x] == credits)
  99                                break;
 100                        if (credit_table[x] > credits) {
 101                                max = x;
 102                        } else {
 103                                if (min == x)
 104                                        break;
 105                                min = x;
 106                        }
 107                }
 108                aeth |= x << IB_AETH_CREDIT_SHIFT;
 109        }
 110        return cpu_to_be32(aeth);
 111}
 112EXPORT_SYMBOL(rvt_compute_aeth);
 113
 114/**
 115 * rvt_get_credit - flush the send work queue of a QP
 116 * @qp: the qp who's send work queue to flush
 117 * @aeth: the Acknowledge Extended Transport Header
 118 *
 119 * The QP s_lock should be held.
 120 */
 121void rvt_get_credit(struct rvt_qp *qp, u32 aeth)
 122{
 123        struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device);
 124        u32 credit = (aeth >> IB_AETH_CREDIT_SHIFT) & IB_AETH_CREDIT_MASK;
 125
 126        lockdep_assert_held(&qp->s_lock);
 127        /*
 128         * If the credit is invalid, we can send
 129         * as many packets as we like.  Otherwise, we have to
 130         * honor the credit field.
 131         */
 132        if (credit == IB_AETH_CREDIT_INVAL) {
 133                if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) {
 134                        qp->s_flags |= RVT_S_UNLIMITED_CREDIT;
 135                        if (qp->s_flags & RVT_S_WAIT_SSN_CREDIT) {
 136                                qp->s_flags &= ~RVT_S_WAIT_SSN_CREDIT;
 137                                rdi->driver_f.schedule_send(qp);
 138                        }
 139                }
 140        } else if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) {
 141                /* Compute new LSN (i.e., MSN + credit) */
 142                credit = (aeth + credit_table[credit]) & IB_MSN_MASK;
 143                if (rvt_cmp_msn(credit, qp->s_lsn) > 0) {
 144                        qp->s_lsn = credit;
 145                        if (qp->s_flags & RVT_S_WAIT_SSN_CREDIT) {
 146                                qp->s_flags &= ~RVT_S_WAIT_SSN_CREDIT;
 147                                rdi->driver_f.schedule_send(qp);
 148                        }
 149                }
 150        }
 151}
 152EXPORT_SYMBOL(rvt_get_credit);
 153
 154/**
 155 * rvt_restart_sge - rewind the sge state for a wqe
 156 * @ss: the sge state pointer
 157 * @wqe: the wqe to rewind
 158 * @len: the data length from the start of the wqe in bytes
 159 *
 160 * Returns the remaining data length.
 161 */
 162u32 rvt_restart_sge(struct rvt_sge_state *ss, struct rvt_swqe *wqe, u32 len)
 163{
 164        ss->sge = wqe->sg_list[0];
 165        ss->sg_list = wqe->sg_list + 1;
 166        ss->num_sge = wqe->wr.num_sge;
 167        ss->total_len = wqe->length;
 168        rvt_skip_sge(ss, len, false);
 169        return wqe->length - len;
 170}
 171EXPORT_SYMBOL(rvt_restart_sge);
 172
 173