1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32#include <linux/dma-mapping.h>
33#include <linux/crc32.h>
34#include <net/ip.h>
35#include <net/ipv6.h>
36#include <net/udp.h>
37#include <linux/iommu.h>
38
39#include <rdma/ib_verbs.h>
40#include <rdma/ib_user_verbs.h>
41#include <rdma/iw_cm.h>
42#include <rdma/ib_umem.h>
43#include <rdma/ib_addr.h>
44#include <rdma/ib_cache.h>
45#include <rdma/uverbs_ioctl.h>
46
47#include <linux/qed/common_hsi.h>
48#include "qedr_hsi_rdma.h"
49#include <linux/qed/qed_if.h>
50#include "qedr.h"
51#include "verbs.h"
52#include <rdma/qedr-abi.h>
53#include "qedr_roce_cm.h"
54
55#define QEDR_SRQ_WQE_ELEM_SIZE sizeof(union rdma_srq_elm)
56#define RDMA_MAX_SGE_PER_SRQ (4)
57#define RDMA_MAX_SRQ_WQE_SIZE (RDMA_MAX_SGE_PER_SRQ + 1)
58
59#define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
60
61static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src,
62 size_t len)
63{
64 size_t min_len = min_t(size_t, len, udata->outlen);
65
66 return ib_copy_to_udata(udata, src, min_len);
67}
68
69int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
70{
71 if (index >= QEDR_ROCE_PKEY_TABLE_LEN)
72 return -EINVAL;
73
74 *pkey = QEDR_ROCE_PKEY_DEFAULT;
75 return 0;
76}
77
78int qedr_iw_query_gid(struct ib_device *ibdev, u8 port,
79 int index, union ib_gid *sgid)
80{
81 struct qedr_dev *dev = get_qedr_dev(ibdev);
82
83 memset(sgid->raw, 0, sizeof(sgid->raw));
84 ether_addr_copy(sgid->raw, dev->ndev->dev_addr);
85
86 DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index,
87 sgid->global.interface_id, sgid->global.subnet_prefix);
88
89 return 0;
90}
91
92int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
93{
94 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
95 struct qedr_device_attr *qattr = &dev->attr;
96 struct qedr_srq *srq = get_qedr_srq(ibsrq);
97
98 srq_attr->srq_limit = srq->srq_limit;
99 srq_attr->max_wr = qattr->max_srq_wr;
100 srq_attr->max_sge = qattr->max_sge;
101
102 return 0;
103}
104
105int qedr_query_device(struct ib_device *ibdev,
106 struct ib_device_attr *attr, struct ib_udata *udata)
107{
108 struct qedr_dev *dev = get_qedr_dev(ibdev);
109 struct qedr_device_attr *qattr = &dev->attr;
110
111 if (!dev->rdma_ctx) {
112 DP_ERR(dev,
113 "qedr_query_device called with invalid params rdma_ctx=%p\n",
114 dev->rdma_ctx);
115 return -EINVAL;
116 }
117
118 memset(attr, 0, sizeof(*attr));
119
120 attr->fw_ver = qattr->fw_ver;
121 attr->sys_image_guid = qattr->sys_image_guid;
122 attr->max_mr_size = qattr->max_mr_size;
123 attr->page_size_cap = qattr->page_size_caps;
124 attr->vendor_id = qattr->vendor_id;
125 attr->vendor_part_id = qattr->vendor_part_id;
126 attr->hw_ver = qattr->hw_ver;
127 attr->max_qp = qattr->max_qp;
128 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
129 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
130 IB_DEVICE_RC_RNR_NAK_GEN |
131 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS;
132
133 attr->max_send_sge = qattr->max_sge;
134 attr->max_recv_sge = qattr->max_sge;
135 attr->max_sge_rd = qattr->max_sge;
136 attr->max_cq = qattr->max_cq;
137 attr->max_cqe = qattr->max_cqe;
138 attr->max_mr = qattr->max_mr;
139 attr->max_mw = qattr->max_mw;
140 attr->max_pd = qattr->max_pd;
141 attr->atomic_cap = dev->atomic_cap;
142 attr->max_fmr = qattr->max_fmr;
143 attr->max_map_per_fmr = 16;
144 attr->max_qp_init_rd_atom =
145 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
146 attr->max_qp_rd_atom =
147 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
148 attr->max_qp_init_rd_atom);
149
150 attr->max_srq = qattr->max_srq;
151 attr->max_srq_sge = qattr->max_srq_sge;
152 attr->max_srq_wr = qattr->max_srq_wr;
153
154 attr->local_ca_ack_delay = qattr->dev_ack_delay;
155 attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
156 attr->max_pkeys = QEDR_ROCE_PKEY_MAX;
157 attr->max_ah = qattr->max_ah;
158
159 return 0;
160}
161
162static inline void get_link_speed_and_width(int speed, u8 *ib_speed,
163 u8 *ib_width)
164{
165 switch (speed) {
166 case 1000:
167 *ib_speed = IB_SPEED_SDR;
168 *ib_width = IB_WIDTH_1X;
169 break;
170 case 10000:
171 *ib_speed = IB_SPEED_QDR;
172 *ib_width = IB_WIDTH_1X;
173 break;
174
175 case 20000:
176 *ib_speed = IB_SPEED_DDR;
177 *ib_width = IB_WIDTH_4X;
178 break;
179
180 case 25000:
181 *ib_speed = IB_SPEED_EDR;
182 *ib_width = IB_WIDTH_1X;
183 break;
184
185 case 40000:
186 *ib_speed = IB_SPEED_QDR;
187 *ib_width = IB_WIDTH_4X;
188 break;
189
190 case 50000:
191 *ib_speed = IB_SPEED_HDR;
192 *ib_width = IB_WIDTH_1X;
193 break;
194
195 case 100000:
196 *ib_speed = IB_SPEED_EDR;
197 *ib_width = IB_WIDTH_4X;
198 break;
199
200 default:
201
202 *ib_speed = IB_SPEED_SDR;
203 *ib_width = IB_WIDTH_1X;
204 }
205}
206
207int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr)
208{
209 struct qedr_dev *dev;
210 struct qed_rdma_port *rdma_port;
211
212 dev = get_qedr_dev(ibdev);
213
214 if (!dev->rdma_ctx) {
215 DP_ERR(dev, "rdma_ctx is NULL\n");
216 return -EINVAL;
217 }
218
219 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
220
221
222 if (rdma_port->port_state == QED_RDMA_PORT_UP) {
223 attr->state = IB_PORT_ACTIVE;
224 attr->phys_state = 5;
225 } else {
226 attr->state = IB_PORT_DOWN;
227 attr->phys_state = 3;
228 }
229 attr->max_mtu = IB_MTU_4096;
230 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
231 attr->lid = 0;
232 attr->lmc = 0;
233 attr->sm_lid = 0;
234 attr->sm_sl = 0;
235 attr->ip_gids = true;
236 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
237 attr->gid_tbl_len = 1;
238 attr->pkey_tbl_len = 1;
239 } else {
240 attr->gid_tbl_len = QEDR_MAX_SGID;
241 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
242 }
243 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
244 attr->qkey_viol_cntr = 0;
245 get_link_speed_and_width(rdma_port->link_speed,
246 &attr->active_speed, &attr->active_width);
247 attr->max_msg_sz = rdma_port->max_msg_size;
248 attr->max_vl_num = 4;
249
250 return 0;
251}
252
253int qedr_modify_port(struct ib_device *ibdev, u8 port, int mask,
254 struct ib_port_modify *props)
255{
256 return 0;
257}
258
259static int qedr_add_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
260 unsigned long len)
261{
262 struct qedr_mm *mm;
263
264 mm = kzalloc(sizeof(*mm), GFP_KERNEL);
265 if (!mm)
266 return -ENOMEM;
267
268 mm->key.phy_addr = phy_addr;
269
270
271
272
273
274
275
276 mm->key.len = roundup(len, PAGE_SIZE);
277 INIT_LIST_HEAD(&mm->entry);
278
279 mutex_lock(&uctx->mm_list_lock);
280 list_add(&mm->entry, &uctx->mm_head);
281 mutex_unlock(&uctx->mm_list_lock);
282
283 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
284 "added (addr=0x%llx,len=0x%lx) for ctx=%p\n",
285 (unsigned long long)mm->key.phy_addr,
286 (unsigned long)mm->key.len, uctx);
287
288 return 0;
289}
290
291static bool qedr_search_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
292 unsigned long len)
293{
294 bool found = false;
295 struct qedr_mm *mm;
296
297 mutex_lock(&uctx->mm_list_lock);
298 list_for_each_entry(mm, &uctx->mm_head, entry) {
299 if (len != mm->key.len || phy_addr != mm->key.phy_addr)
300 continue;
301
302 found = true;
303 break;
304 }
305 mutex_unlock(&uctx->mm_list_lock);
306 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
307 "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, result=%d\n",
308 mm->key.phy_addr, mm->key.len, uctx, found);
309
310 return found;
311}
312
313int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
314{
315 struct ib_device *ibdev = uctx->device;
316 int rc;
317 struct qedr_ucontext *ctx = get_qedr_ucontext(uctx);
318 struct qedr_alloc_ucontext_resp uresp = {};
319 struct qedr_dev *dev = get_qedr_dev(ibdev);
320 struct qed_rdma_add_user_out_params oparams;
321
322 if (!udata)
323 return -EFAULT;
324
325 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
326 if (rc) {
327 DP_ERR(dev,
328 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
329 rc);
330 return rc;
331 }
332
333 ctx->dpi = oparams.dpi;
334 ctx->dpi_addr = oparams.dpi_addr;
335 ctx->dpi_phys_addr = oparams.dpi_phys_addr;
336 ctx->dpi_size = oparams.dpi_size;
337 INIT_LIST_HEAD(&ctx->mm_head);
338 mutex_init(&ctx->mm_list_lock);
339
340 uresp.dpm_enabled = dev->user_dpm_enabled;
341 uresp.wids_enabled = 1;
342 uresp.wid_count = oparams.wid_count;
343 uresp.db_pa = ctx->dpi_phys_addr;
344 uresp.db_size = ctx->dpi_size;
345 uresp.max_send_wr = dev->attr.max_sqe;
346 uresp.max_recv_wr = dev->attr.max_rqe;
347 uresp.max_srq_wr = dev->attr.max_srq_wr;
348 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
349 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
350 uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
351 uresp.max_cqes = QEDR_MAX_CQES;
352
353 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
354 if (rc)
355 return rc;
356
357 ctx->dev = dev;
358
359 rc = qedr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size);
360 if (rc)
361 return rc;
362
363 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
364 &ctx->ibucontext);
365 return 0;
366}
367
368void qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
369{
370 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
371 struct qedr_mm *mm, *tmp;
372
373 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
374 uctx);
375 uctx->dev->ops->rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi);
376
377 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
378 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
379 "deleted (addr=0x%llx,len=0x%lx) for ctx=%p\n",
380 mm->key.phy_addr, mm->key.len, uctx);
381 list_del(&mm->entry);
382 kfree(mm);
383 }
384}
385
386int qedr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
387{
388 struct qedr_ucontext *ucontext = get_qedr_ucontext(context);
389 struct qedr_dev *dev = get_qedr_dev(context->device);
390 unsigned long phys_addr = vma->vm_pgoff << PAGE_SHIFT;
391 unsigned long len = (vma->vm_end - vma->vm_start);
392 unsigned long dpi_start;
393
394 dpi_start = dev->db_phys_addr + (ucontext->dpi * ucontext->dpi_size);
395
396 DP_DEBUG(dev, QEDR_MSG_INIT,
397 "mmap invoked with vm_start=0x%pK, vm_end=0x%pK,vm_pgoff=0x%pK; dpi_start=0x%pK dpi_size=0x%x\n",
398 (void *)vma->vm_start, (void *)vma->vm_end,
399 (void *)vma->vm_pgoff, (void *)dpi_start, ucontext->dpi_size);
400
401 if ((vma->vm_start & (PAGE_SIZE - 1)) || (len & (PAGE_SIZE - 1))) {
402 DP_ERR(dev,
403 "failed mmap, addresses must be page aligned: start=0x%pK, end=0x%pK\n",
404 (void *)vma->vm_start, (void *)vma->vm_end);
405 return -EINVAL;
406 }
407
408 if (!qedr_search_mmap(ucontext, phys_addr, len)) {
409 DP_ERR(dev, "failed mmap, vm_pgoff=0x%lx is not authorized\n",
410 vma->vm_pgoff);
411 return -EINVAL;
412 }
413
414 if (phys_addr < dpi_start ||
415 ((phys_addr + len) > (dpi_start + ucontext->dpi_size))) {
416 DP_ERR(dev,
417 "failed mmap, pages are outside of dpi; page address=0x%pK, dpi_start=0x%pK, dpi_size=0x%x\n",
418 (void *)phys_addr, (void *)dpi_start,
419 ucontext->dpi_size);
420 return -EINVAL;
421 }
422
423 if (vma->vm_flags & VM_READ) {
424 DP_ERR(dev, "failed mmap, cannot map doorbell bar for read\n");
425 return -EINVAL;
426 }
427
428 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
429 return io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, len,
430 vma->vm_page_prot);
431}
432
433int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
434{
435 struct ib_device *ibdev = ibpd->device;
436 struct qedr_dev *dev = get_qedr_dev(ibdev);
437 struct qedr_pd *pd = get_qedr_pd(ibpd);
438 u16 pd_id;
439 int rc;
440
441 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
442 udata ? "User Lib" : "Kernel");
443
444 if (!dev->rdma_ctx) {
445 DP_ERR(dev, "invalid RDMA context\n");
446 return -EINVAL;
447 }
448
449 rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
450 if (rc)
451 return rc;
452
453 pd->pd_id = pd_id;
454
455 if (udata) {
456 struct qedr_alloc_pd_uresp uresp = {
457 .pd_id = pd_id,
458 };
459 struct qedr_ucontext *context = rdma_udata_to_drv_context(
460 udata, struct qedr_ucontext, ibucontext);
461
462 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
463 if (rc) {
464 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
465 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
466 return rc;
467 }
468
469 pd->uctx = context;
470 pd->uctx->pd = pd;
471 }
472
473 return 0;
474}
475
476void qedr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
477{
478 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
479 struct qedr_pd *pd = get_qedr_pd(ibpd);
480
481 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
482 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
483}
484
485static void qedr_free_pbl(struct qedr_dev *dev,
486 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
487{
488 struct pci_dev *pdev = dev->pdev;
489 int i;
490
491 for (i = 0; i < pbl_info->num_pbls; i++) {
492 if (!pbl[i].va)
493 continue;
494 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
495 pbl[i].va, pbl[i].pa);
496 }
497
498 kfree(pbl);
499}
500
501#define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
502#define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
503
504#define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
505#define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
506#define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
507
508static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
509 struct qedr_pbl_info *pbl_info,
510 gfp_t flags)
511{
512 struct pci_dev *pdev = dev->pdev;
513 struct qedr_pbl *pbl_table;
514 dma_addr_t *pbl_main_tbl;
515 dma_addr_t pa;
516 void *va;
517 int i;
518
519 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
520 if (!pbl_table)
521 return ERR_PTR(-ENOMEM);
522
523 for (i = 0; i < pbl_info->num_pbls; i++) {
524 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, &pa,
525 flags);
526 if (!va)
527 goto err;
528
529 pbl_table[i].va = va;
530 pbl_table[i].pa = pa;
531 }
532
533
534
535
536 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
537 for (i = 0; i < pbl_info->num_pbls - 1; i++)
538 pbl_main_tbl[i] = pbl_table[i + 1].pa;
539
540 return pbl_table;
541
542err:
543 for (i--; i >= 0; i--)
544 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
545 pbl_table[i].va, pbl_table[i].pa);
546
547 qedr_free_pbl(dev, pbl_info, pbl_table);
548
549 return ERR_PTR(-ENOMEM);
550}
551
552static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
553 struct qedr_pbl_info *pbl_info,
554 u32 num_pbes, int two_layer_capable)
555{
556 u32 pbl_capacity;
557 u32 pbl_size;
558 u32 num_pbls;
559
560 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
561 if (num_pbes > MAX_PBES_TWO_LAYER) {
562 DP_ERR(dev, "prepare pbl table: too many pages %d\n",
563 num_pbes);
564 return -EINVAL;
565 }
566
567
568 pbl_size = MIN_FW_PBL_PAGE_SIZE;
569 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
570 NUM_PBES_ON_PAGE(pbl_size);
571
572 while (pbl_capacity < num_pbes) {
573 pbl_size *= 2;
574 pbl_capacity = pbl_size / sizeof(u64);
575 pbl_capacity = pbl_capacity * pbl_capacity;
576 }
577
578 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
579 num_pbls++;
580 pbl_info->two_layered = true;
581 } else {
582
583 num_pbls = 1;
584 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
585 roundup_pow_of_two((num_pbes * sizeof(u64))));
586 pbl_info->two_layered = false;
587 }
588
589 pbl_info->num_pbls = num_pbls;
590 pbl_info->pbl_size = pbl_size;
591 pbl_info->num_pbes = num_pbes;
592
593 DP_DEBUG(dev, QEDR_MSG_MR,
594 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
595 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
596
597 return 0;
598}
599
600static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
601 struct qedr_pbl *pbl,
602 struct qedr_pbl_info *pbl_info, u32 pg_shift)
603{
604 int pbe_cnt, total_num_pbes = 0;
605 u32 fw_pg_cnt, fw_pg_per_umem_pg;
606 struct qedr_pbl *pbl_tbl;
607 struct sg_dma_page_iter sg_iter;
608 struct regpair *pbe;
609 u64 pg_addr;
610
611 if (!pbl_info->num_pbes)
612 return;
613
614
615
616
617 if (pbl_info->two_layered)
618 pbl_tbl = &pbl[1];
619 else
620 pbl_tbl = pbl;
621
622 pbe = (struct regpair *)pbl_tbl->va;
623 if (!pbe) {
624 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
625 return;
626 }
627
628 pbe_cnt = 0;
629
630 fw_pg_per_umem_pg = BIT(PAGE_SHIFT - pg_shift);
631
632 for_each_sg_dma_page (umem->sg_head.sgl, &sg_iter, umem->nmap, 0) {
633 pg_addr = sg_page_iter_dma_address(&sg_iter);
634 for (fw_pg_cnt = 0; fw_pg_cnt < fw_pg_per_umem_pg;) {
635 pbe->lo = cpu_to_le32(pg_addr);
636 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));
637
638 pg_addr += BIT(pg_shift);
639 pbe_cnt++;
640 total_num_pbes++;
641 pbe++;
642
643 if (total_num_pbes == pbl_info->num_pbes)
644 return;
645
646
647
648
649 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
650 pbl_tbl++;
651 pbe = (struct regpair *)pbl_tbl->va;
652 pbe_cnt = 0;
653 }
654
655 fw_pg_cnt++;
656 }
657 }
658}
659
660static int qedr_copy_cq_uresp(struct qedr_dev *dev,
661 struct qedr_cq *cq, struct ib_udata *udata)
662{
663 struct qedr_create_cq_uresp uresp;
664 int rc;
665
666 memset(&uresp, 0, sizeof(uresp));
667
668 uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
669 uresp.icid = cq->icid;
670
671 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
672 if (rc)
673 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
674
675 return rc;
676}
677
678static void consume_cqe(struct qedr_cq *cq)
679{
680 if (cq->latest_cqe == cq->toggle_cqe)
681 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
682
683 cq->latest_cqe = qed_chain_consume(&cq->pbl);
684}
685
686static inline int qedr_align_cq_entries(int entries)
687{
688 u64 size, aligned_size;
689
690
691 size = (entries + 1) * QEDR_CQE_SIZE;
692 aligned_size = ALIGN(size, PAGE_SIZE);
693
694 return aligned_size / QEDR_CQE_SIZE;
695}
696
697static inline int qedr_init_user_queue(struct ib_udata *udata,
698 struct qedr_dev *dev,
699 struct qedr_userq *q, u64 buf_addr,
700 size_t buf_len, int access, int dmasync,
701 int alloc_and_init)
702{
703 u32 fw_pages;
704 int rc;
705
706 q->buf_addr = buf_addr;
707 q->buf_len = buf_len;
708 q->umem = ib_umem_get(udata, q->buf_addr, q->buf_len, access, dmasync);
709 if (IS_ERR(q->umem)) {
710 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
711 PTR_ERR(q->umem));
712 return PTR_ERR(q->umem);
713 }
714
715 fw_pages = ib_umem_page_count(q->umem) <<
716 (PAGE_SHIFT - FW_PAGE_SHIFT);
717
718 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);
719 if (rc)
720 goto err0;
721
722 if (alloc_and_init) {
723 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
724 if (IS_ERR(q->pbl_tbl)) {
725 rc = PTR_ERR(q->pbl_tbl);
726 goto err0;
727 }
728 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,
729 FW_PAGE_SHIFT);
730 } else {
731 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);
732 if (!q->pbl_tbl) {
733 rc = -ENOMEM;
734 goto err0;
735 }
736 }
737
738 return 0;
739
740err0:
741 ib_umem_release(q->umem);
742 q->umem = NULL;
743
744 return rc;
745}
746
747static inline void qedr_init_cq_params(struct qedr_cq *cq,
748 struct qedr_ucontext *ctx,
749 struct qedr_dev *dev, int vector,
750 int chain_entries, int page_cnt,
751 u64 pbl_ptr,
752 struct qed_rdma_create_cq_in_params
753 *params)
754{
755 memset(params, 0, sizeof(*params));
756 params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
757 params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
758 params->cnq_id = vector;
759 params->cq_size = chain_entries - 1;
760 params->dpi = (ctx) ? ctx->dpi : dev->dpi;
761 params->pbl_num_pages = page_cnt;
762 params->pbl_ptr = pbl_ptr;
763 params->pbl_two_level = 0;
764}
765
766static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
767{
768 cq->db.data.agg_flags = flags;
769 cq->db.data.value = cpu_to_le32(cons);
770 writeq(cq->db.raw, cq->db_addr);
771}
772
773int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
774{
775 struct qedr_cq *cq = get_qedr_cq(ibcq);
776 unsigned long sflags;
777 struct qedr_dev *dev;
778
779 dev = get_qedr_dev(ibcq->device);
780
781 if (cq->destroyed) {
782 DP_ERR(dev,
783 "warning: arm was invoked after destroy for cq %p (icid=%d)\n",
784 cq, cq->icid);
785 return -EINVAL;
786 }
787
788
789 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
790 return 0;
791
792 spin_lock_irqsave(&cq->cq_lock, sflags);
793
794 cq->arm_flags = 0;
795
796 if (flags & IB_CQ_SOLICITED)
797 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
798
799 if (flags & IB_CQ_NEXT_COMP)
800 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
801
802 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
803
804 spin_unlock_irqrestore(&cq->cq_lock, sflags);
805
806 return 0;
807}
808
809int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
810 struct ib_udata *udata)
811{
812 struct ib_device *ibdev = ibcq->device;
813 struct qedr_ucontext *ctx = rdma_udata_to_drv_context(
814 udata, struct qedr_ucontext, ibucontext);
815 struct qed_rdma_destroy_cq_out_params destroy_oparams;
816 struct qed_rdma_destroy_cq_in_params destroy_iparams;
817 struct qedr_dev *dev = get_qedr_dev(ibdev);
818 struct qed_rdma_create_cq_in_params params;
819 struct qedr_create_cq_ureq ureq = {};
820 int vector = attr->comp_vector;
821 int entries = attr->cqe;
822 struct qedr_cq *cq = get_qedr_cq(ibcq);
823 int chain_entries;
824 int page_cnt;
825 u64 pbl_ptr;
826 u16 icid;
827 int rc;
828
829 DP_DEBUG(dev, QEDR_MSG_INIT,
830 "create_cq: called from %s. entries=%d, vector=%d\n",
831 udata ? "User Lib" : "Kernel", entries, vector);
832
833 if (entries > QEDR_MAX_CQES) {
834 DP_ERR(dev,
835 "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
836 entries, QEDR_MAX_CQES);
837 return -EINVAL;
838 }
839
840 chain_entries = qedr_align_cq_entries(entries);
841 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
842
843 if (udata) {
844 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
845 DP_ERR(dev,
846 "create cq: problem copying data from user space\n");
847 goto err0;
848 }
849
850 if (!ureq.len) {
851 DP_ERR(dev,
852 "create cq: cannot create a cq with 0 entries\n");
853 goto err0;
854 }
855
856 cq->cq_type = QEDR_CQ_TYPE_USER;
857
858 rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr,
859 ureq.len, IB_ACCESS_LOCAL_WRITE, 1,
860 1);
861 if (rc)
862 goto err0;
863
864 pbl_ptr = cq->q.pbl_tbl->pa;
865 page_cnt = cq->q.pbl_info.num_pbes;
866
867 cq->ibcq.cqe = chain_entries;
868 } else {
869 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
870
871 rc = dev->ops->common->chain_alloc(dev->cdev,
872 QED_CHAIN_USE_TO_CONSUME,
873 QED_CHAIN_MODE_PBL,
874 QED_CHAIN_CNT_TYPE_U32,
875 chain_entries,
876 sizeof(union rdma_cqe),
877 &cq->pbl, NULL);
878 if (rc)
879 goto err1;
880
881 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
882 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
883 cq->ibcq.cqe = cq->pbl.capacity;
884 }
885
886 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
887 pbl_ptr, ¶ms);
888
889 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, ¶ms, &icid);
890 if (rc)
891 goto err2;
892
893 cq->icid = icid;
894 cq->sig = QEDR_CQ_MAGIC_NUMBER;
895 spin_lock_init(&cq->cq_lock);
896
897 if (udata) {
898 rc = qedr_copy_cq_uresp(dev, cq, udata);
899 if (rc)
900 goto err3;
901 } else {
902
903 cq->db_addr = dev->db_addr +
904 DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
905 cq->db.data.icid = cq->icid;
906 cq->db.data.params = DB_AGG_CMD_SET <<
907 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
908
909
910 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
911 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
912 cq->latest_cqe = NULL;
913 consume_cqe(cq);
914 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
915 }
916
917 DP_DEBUG(dev, QEDR_MSG_CQ,
918 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
919 cq->icid, cq, params.cq_size);
920
921 return 0;
922
923err3:
924 destroy_iparams.icid = cq->icid;
925 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
926 &destroy_oparams);
927err2:
928 if (udata)
929 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
930 else
931 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
932err1:
933 if (udata)
934 ib_umem_release(cq->q.umem);
935err0:
936 return -EINVAL;
937}
938
939int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
940{
941 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
942 struct qedr_cq *cq = get_qedr_cq(ibcq);
943
944 DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq);
945
946 return 0;
947}
948
949#define QEDR_DESTROY_CQ_MAX_ITERATIONS (10)
950#define QEDR_DESTROY_CQ_ITER_DURATION (10)
951
952void qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
953{
954 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
955 struct qed_rdma_destroy_cq_out_params oparams;
956 struct qed_rdma_destroy_cq_in_params iparams;
957 struct qedr_cq *cq = get_qedr_cq(ibcq);
958 int iter;
959
960 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
961
962 cq->destroyed = 1;
963
964
965 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
966 return;
967
968 iparams.icid = cq->icid;
969 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
970 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
971
972 if (udata) {
973 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
974 ib_umem_release(cq->q.umem);
975 }
976
977
978
979
980
981
982
983
984
985
986
987
988 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
989 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
990 udelay(QEDR_DESTROY_CQ_ITER_DURATION);
991 iter--;
992 }
993
994 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
995 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
996 msleep(QEDR_DESTROY_CQ_ITER_DURATION);
997 iter--;
998 }
999
1000
1001
1002
1003
1004
1005}
1006
1007static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1008 struct ib_qp_attr *attr,
1009 int attr_mask,
1010 struct qed_rdma_modify_qp_in_params
1011 *qp_params)
1012{
1013 const struct ib_gid_attr *gid_attr;
1014 enum rdma_network_type nw_type;
1015 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
1016 u32 ipv4_addr;
1017 int ret;
1018 int i;
1019
1020 gid_attr = grh->sgid_attr;
1021 ret = rdma_read_gid_l2_fields(gid_attr, &qp_params->vlan_id, NULL);
1022 if (ret)
1023 return ret;
1024
1025 nw_type = rdma_gid_attr_network_type(gid_attr);
1026 switch (nw_type) {
1027 case RDMA_NETWORK_IPV6:
1028 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1029 sizeof(qp_params->sgid));
1030 memcpy(&qp_params->dgid.bytes[0],
1031 &grh->dgid,
1032 sizeof(qp_params->dgid));
1033 qp_params->roce_mode = ROCE_V2_IPV6;
1034 SET_FIELD(qp_params->modify_flags,
1035 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1036 break;
1037 case RDMA_NETWORK_IB:
1038 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1039 sizeof(qp_params->sgid));
1040 memcpy(&qp_params->dgid.bytes[0],
1041 &grh->dgid,
1042 sizeof(qp_params->dgid));
1043 qp_params->roce_mode = ROCE_V1;
1044 break;
1045 case RDMA_NETWORK_IPV4:
1046 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1047 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
1048 ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw);
1049 qp_params->sgid.ipv4_addr = ipv4_addr;
1050 ipv4_addr =
1051 qedr_get_ipv4_from_gid(grh->dgid.raw);
1052 qp_params->dgid.ipv4_addr = ipv4_addr;
1053 SET_FIELD(qp_params->modify_flags,
1054 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1055 qp_params->roce_mode = ROCE_V2_IPV4;
1056 break;
1057 }
1058
1059 for (i = 0; i < 4; i++) {
1060 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1061 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1062 }
1063
1064 if (qp_params->vlan_id >= VLAN_CFI_MASK)
1065 qp_params->vlan_id = 0;
1066
1067 return 0;
1068}
1069
1070static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1071 struct ib_qp_init_attr *attrs,
1072 struct ib_udata *udata)
1073{
1074 struct qedr_device_attr *qattr = &dev->attr;
1075
1076
1077 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
1078 DP_DEBUG(dev, QEDR_MSG_QP,
1079 "create qp: unsupported qp type=0x%x requested\n",
1080 attrs->qp_type);
1081 return -EINVAL;
1082 }
1083
1084 if (attrs->cap.max_send_wr > qattr->max_sqe) {
1085 DP_ERR(dev,
1086 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1087 attrs->cap.max_send_wr, qattr->max_sqe);
1088 return -EINVAL;
1089 }
1090
1091 if (attrs->cap.max_inline_data > qattr->max_inline) {
1092 DP_ERR(dev,
1093 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1094 attrs->cap.max_inline_data, qattr->max_inline);
1095 return -EINVAL;
1096 }
1097
1098 if (attrs->cap.max_send_sge > qattr->max_sge) {
1099 DP_ERR(dev,
1100 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1101 attrs->cap.max_send_sge, qattr->max_sge);
1102 return -EINVAL;
1103 }
1104
1105 if (attrs->cap.max_recv_sge > qattr->max_sge) {
1106 DP_ERR(dev,
1107 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1108 attrs->cap.max_recv_sge, qattr->max_sge);
1109 return -EINVAL;
1110 }
1111
1112
1113 if (udata && attrs->qp_type == IB_QPT_GSI) {
1114 DP_ERR(dev,
1115 "create qp: userspace can't create special QPs of type=0x%x\n",
1116 attrs->qp_type);
1117 return -EINVAL;
1118 }
1119
1120 return 0;
1121}
1122
1123static int qedr_copy_srq_uresp(struct qedr_dev *dev,
1124 struct qedr_srq *srq, struct ib_udata *udata)
1125{
1126 struct qedr_create_srq_uresp uresp = {};
1127 int rc;
1128
1129 uresp.srq_id = srq->srq_id;
1130
1131 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1132 if (rc)
1133 DP_ERR(dev, "create srq: problem copying data to user space\n");
1134
1135 return rc;
1136}
1137
1138static void qedr_copy_rq_uresp(struct qedr_dev *dev,
1139 struct qedr_create_qp_uresp *uresp,
1140 struct qedr_qp *qp)
1141{
1142
1143 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1144 uresp->rq_db_offset =
1145 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1146 uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1147 } else {
1148 uresp->rq_db_offset =
1149 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1150 }
1151
1152 uresp->rq_icid = qp->icid;
1153}
1154
1155static void qedr_copy_sq_uresp(struct qedr_dev *dev,
1156 struct qedr_create_qp_uresp *uresp,
1157 struct qedr_qp *qp)
1158{
1159 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1160
1161
1162 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1163 uresp->sq_icid = qp->icid;
1164 else
1165 uresp->sq_icid = qp->icid + 1;
1166}
1167
1168static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1169 struct qedr_qp *qp, struct ib_udata *udata)
1170{
1171 struct qedr_create_qp_uresp uresp;
1172 int rc;
1173
1174 memset(&uresp, 0, sizeof(uresp));
1175 qedr_copy_sq_uresp(dev, &uresp, qp);
1176 qedr_copy_rq_uresp(dev, &uresp, qp);
1177
1178 uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1179 uresp.qp_id = qp->qp_id;
1180
1181 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1182 if (rc)
1183 DP_ERR(dev,
1184 "create qp: failed a copy to user space with qp icid=0x%x.\n",
1185 qp->icid);
1186
1187 return rc;
1188}
1189
1190static void qedr_set_common_qp_params(struct qedr_dev *dev,
1191 struct qedr_qp *qp,
1192 struct qedr_pd *pd,
1193 struct ib_qp_init_attr *attrs)
1194{
1195 spin_lock_init(&qp->q_lock);
1196 atomic_set(&qp->refcnt, 1);
1197 qp->pd = pd;
1198 qp->qp_type = attrs->qp_type;
1199 qp->max_inline_data = attrs->cap.max_inline_data;
1200 qp->sq.max_sges = attrs->cap.max_send_sge;
1201 qp->state = QED_ROCE_QP_STATE_RESET;
1202 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1203 qp->sq_cq = get_qedr_cq(attrs->send_cq);
1204 qp->dev = dev;
1205
1206 if (attrs->srq) {
1207 qp->srq = get_qedr_srq(attrs->srq);
1208 } else {
1209 qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1210 qp->rq.max_sges = attrs->cap.max_recv_sge;
1211 DP_DEBUG(dev, QEDR_MSG_QP,
1212 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1213 qp->rq.max_sges, qp->rq_cq->icid);
1214 }
1215
1216 DP_DEBUG(dev, QEDR_MSG_QP,
1217 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1218 pd->pd_id, qp->qp_type, qp->max_inline_data,
1219 qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1220 DP_DEBUG(dev, QEDR_MSG_QP,
1221 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1222 qp->sq.max_sges, qp->sq_cq->icid);
1223}
1224
1225static void qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1226{
1227 qp->sq.db = dev->db_addr +
1228 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1229 qp->sq.db_data.data.icid = qp->icid + 1;
1230 if (!qp->srq) {
1231 qp->rq.db = dev->db_addr +
1232 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1233 qp->rq.db_data.data.icid = qp->icid;
1234 }
1235}
1236
1237static int qedr_check_srq_params(struct qedr_dev *dev,
1238 struct ib_srq_init_attr *attrs,
1239 struct ib_udata *udata)
1240{
1241 struct qedr_device_attr *qattr = &dev->attr;
1242
1243 if (attrs->attr.max_wr > qattr->max_srq_wr) {
1244 DP_ERR(dev,
1245 "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n",
1246 attrs->attr.max_wr, qattr->max_srq_wr);
1247 return -EINVAL;
1248 }
1249
1250 if (attrs->attr.max_sge > qattr->max_sge) {
1251 DP_ERR(dev,
1252 "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n",
1253 attrs->attr.max_sge, qattr->max_sge);
1254 return -EINVAL;
1255 }
1256
1257 return 0;
1258}
1259
1260static void qedr_free_srq_user_params(struct qedr_srq *srq)
1261{
1262 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1263 ib_umem_release(srq->usrq.umem);
1264 ib_umem_release(srq->prod_umem);
1265}
1266
1267static void qedr_free_srq_kernel_params(struct qedr_srq *srq)
1268{
1269 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1270 struct qedr_dev *dev = srq->dev;
1271
1272 dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl);
1273
1274 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1275 hw_srq->virt_prod_pair_addr,
1276 hw_srq->phy_prod_pair_addr);
1277}
1278
1279static int qedr_init_srq_user_params(struct ib_udata *udata,
1280 struct qedr_srq *srq,
1281 struct qedr_create_srq_ureq *ureq,
1282 int access, int dmasync)
1283{
1284 struct scatterlist *sg;
1285 int rc;
1286
1287 rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr,
1288 ureq->srq_len, access, dmasync, 1);
1289 if (rc)
1290 return rc;
1291
1292 srq->prod_umem =
1293 ib_umem_get(udata, ureq->prod_pair_addr,
1294 sizeof(struct rdma_srq_producers), access, dmasync);
1295 if (IS_ERR(srq->prod_umem)) {
1296 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1297 ib_umem_release(srq->usrq.umem);
1298 DP_ERR(srq->dev,
1299 "create srq: failed ib_umem_get for producer, got %ld\n",
1300 PTR_ERR(srq->prod_umem));
1301 return PTR_ERR(srq->prod_umem);
1302 }
1303
1304 sg = srq->prod_umem->sg_head.sgl;
1305 srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);
1306
1307 return 0;
1308}
1309
1310static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq,
1311 struct qedr_dev *dev,
1312 struct ib_srq_init_attr *init_attr)
1313{
1314 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1315 dma_addr_t phy_prod_pair_addr;
1316 u32 num_elems;
1317 void *va;
1318 int rc;
1319
1320 va = dma_alloc_coherent(&dev->pdev->dev,
1321 sizeof(struct rdma_srq_producers),
1322 &phy_prod_pair_addr, GFP_KERNEL);
1323 if (!va) {
1324 DP_ERR(dev,
1325 "create srq: failed to allocate dma memory for producer\n");
1326 return -ENOMEM;
1327 }
1328
1329 hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;
1330 hw_srq->virt_prod_pair_addr = va;
1331
1332 num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE;
1333 rc = dev->ops->common->chain_alloc(dev->cdev,
1334 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1335 QED_CHAIN_MODE_PBL,
1336 QED_CHAIN_CNT_TYPE_U32,
1337 num_elems,
1338 QEDR_SRQ_WQE_ELEM_SIZE,
1339 &hw_srq->pbl, NULL);
1340 if (rc)
1341 goto err0;
1342
1343 hw_srq->num_elems = num_elems;
1344
1345 return 0;
1346
1347err0:
1348 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1349 va, phy_prod_pair_addr);
1350 return rc;
1351}
1352
1353int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
1354 struct ib_udata *udata)
1355{
1356 struct qed_rdma_destroy_srq_in_params destroy_in_params;
1357 struct qed_rdma_create_srq_in_params in_params = {};
1358 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1359 struct qed_rdma_create_srq_out_params out_params;
1360 struct qedr_pd *pd = get_qedr_pd(ibsrq->pd);
1361 struct qedr_create_srq_ureq ureq = {};
1362 u64 pbl_base_addr, phy_prod_pair_addr;
1363 struct qedr_srq_hwq_info *hw_srq;
1364 u32 page_cnt, page_size;
1365 struct qedr_srq *srq = get_qedr_srq(ibsrq);
1366 int rc = 0;
1367
1368 DP_DEBUG(dev, QEDR_MSG_QP,
1369 "create SRQ called from %s (pd %p)\n",
1370 (udata) ? "User lib" : "kernel", pd);
1371
1372 rc = qedr_check_srq_params(dev, init_attr, udata);
1373 if (rc)
1374 return -EINVAL;
1375
1376 srq->dev = dev;
1377 hw_srq = &srq->hw_srq;
1378 spin_lock_init(&srq->lock);
1379
1380 hw_srq->max_wr = init_attr->attr.max_wr;
1381 hw_srq->max_sges = init_attr->attr.max_sge;
1382
1383 if (udata) {
1384 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
1385 DP_ERR(dev,
1386 "create srq: problem copying data from user space\n");
1387 goto err0;
1388 }
1389
1390 rc = qedr_init_srq_user_params(udata, srq, &ureq, 0, 0);
1391 if (rc)
1392 goto err0;
1393
1394 page_cnt = srq->usrq.pbl_info.num_pbes;
1395 pbl_base_addr = srq->usrq.pbl_tbl->pa;
1396 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1397 page_size = PAGE_SIZE;
1398 } else {
1399 struct qed_chain *pbl;
1400
1401 rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr);
1402 if (rc)
1403 goto err0;
1404
1405 pbl = &hw_srq->pbl;
1406 page_cnt = qed_chain_get_page_cnt(pbl);
1407 pbl_base_addr = qed_chain_get_pbl_phys(pbl);
1408 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1409 page_size = QED_CHAIN_PAGE_SIZE;
1410 }
1411
1412 in_params.pd_id = pd->pd_id;
1413 in_params.pbl_base_addr = pbl_base_addr;
1414 in_params.prod_pair_addr = phy_prod_pair_addr;
1415 in_params.num_pages = page_cnt;
1416 in_params.page_size = page_size;
1417
1418 rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);
1419 if (rc)
1420 goto err1;
1421
1422 srq->srq_id = out_params.srq_id;
1423
1424 if (udata) {
1425 rc = qedr_copy_srq_uresp(dev, srq, udata);
1426 if (rc)
1427 goto err2;
1428 }
1429
1430 rc = xa_insert_irq(&dev->srqs, srq->srq_id, srq, GFP_KERNEL);
1431 if (rc)
1432 goto err2;
1433
1434 DP_DEBUG(dev, QEDR_MSG_SRQ,
1435 "create srq: created srq with srq_id=0x%0x\n", srq->srq_id);
1436 return 0;
1437
1438err2:
1439 destroy_in_params.srq_id = srq->srq_id;
1440
1441 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);
1442err1:
1443 if (udata)
1444 qedr_free_srq_user_params(srq);
1445 else
1446 qedr_free_srq_kernel_params(srq);
1447err0:
1448 return -EFAULT;
1449}
1450
1451void qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
1452{
1453 struct qed_rdma_destroy_srq_in_params in_params = {};
1454 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1455 struct qedr_srq *srq = get_qedr_srq(ibsrq);
1456
1457 xa_erase_irq(&dev->srqs, srq->srq_id);
1458 in_params.srq_id = srq->srq_id;
1459 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params);
1460
1461 if (ibsrq->uobject)
1462 qedr_free_srq_user_params(srq);
1463 else
1464 qedr_free_srq_kernel_params(srq);
1465
1466 DP_DEBUG(dev, QEDR_MSG_SRQ,
1467 "destroy srq: destroyed srq with srq_id=0x%0x\n",
1468 srq->srq_id);
1469}
1470
1471int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
1472 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1473{
1474 struct qed_rdma_modify_srq_in_params in_params = {};
1475 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1476 struct qedr_srq *srq = get_qedr_srq(ibsrq);
1477 int rc;
1478
1479 if (attr_mask & IB_SRQ_MAX_WR) {
1480 DP_ERR(dev,
1481 "modify srq: invalid attribute mask=0x%x specified for %p\n",
1482 attr_mask, srq);
1483 return -EINVAL;
1484 }
1485
1486 if (attr_mask & IB_SRQ_LIMIT) {
1487 if (attr->srq_limit >= srq->hw_srq.max_wr) {
1488 DP_ERR(dev,
1489 "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n",
1490 attr->srq_limit, srq->hw_srq.max_wr);
1491 return -EINVAL;
1492 }
1493
1494 in_params.srq_id = srq->srq_id;
1495 in_params.wqe_limit = attr->srq_limit;
1496 rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params);
1497 if (rc)
1498 return rc;
1499 }
1500
1501 srq->srq_limit = attr->srq_limit;
1502
1503 DP_DEBUG(dev, QEDR_MSG_SRQ,
1504 "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id);
1505
1506 return 0;
1507}
1508
1509static inline void
1510qedr_init_common_qp_in_params(struct qedr_dev *dev,
1511 struct qedr_pd *pd,
1512 struct qedr_qp *qp,
1513 struct ib_qp_init_attr *attrs,
1514 bool fmr_and_reserved_lkey,
1515 struct qed_rdma_create_qp_in_params *params)
1516{
1517
1518 params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1519 params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1520
1521 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1522 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1523 params->pd = pd->pd_id;
1524 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1525 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1526 params->stats_queue = 0;
1527 params->srq_id = 0;
1528 params->use_srq = false;
1529
1530 if (!qp->srq) {
1531 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1532
1533 } else {
1534 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1535 params->srq_id = qp->srq->srq_id;
1536 params->use_srq = true;
1537 }
1538}
1539
1540static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
1541{
1542 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1543 "qp=%p. "
1544 "sq_addr=0x%llx, "
1545 "sq_len=%zd, "
1546 "rq_addr=0x%llx, "
1547 "rq_len=%zd"
1548 "\n",
1549 qp,
1550 qp->usq.buf_addr,
1551 qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len);
1552}
1553
1554static inline void
1555qedr_iwarp_populate_user_qp(struct qedr_dev *dev,
1556 struct qedr_qp *qp,
1557 struct qed_rdma_create_qp_out_params *out_params)
1558{
1559 qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
1560 qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
1561
1562 qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
1563 &qp->usq.pbl_info, FW_PAGE_SHIFT);
1564 if (!qp->srq) {
1565 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
1566 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
1567 }
1568
1569 qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
1570 &qp->urq.pbl_info, FW_PAGE_SHIFT);
1571}
1572
1573static void qedr_cleanup_user(struct qedr_dev *dev, struct qedr_qp *qp)
1574{
1575 ib_umem_release(qp->usq.umem);
1576 qp->usq.umem = NULL;
1577
1578 ib_umem_release(qp->urq.umem);
1579 qp->urq.umem = NULL;
1580}
1581
1582static int qedr_create_user_qp(struct qedr_dev *dev,
1583 struct qedr_qp *qp,
1584 struct ib_pd *ibpd,
1585 struct ib_udata *udata,
1586 struct ib_qp_init_attr *attrs)
1587{
1588 struct qed_rdma_create_qp_in_params in_params;
1589 struct qed_rdma_create_qp_out_params out_params;
1590 struct qedr_pd *pd = get_qedr_pd(ibpd);
1591 struct qedr_create_qp_ureq ureq;
1592 int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1);
1593 int rc = -EINVAL;
1594
1595 memset(&ureq, 0, sizeof(ureq));
1596 rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
1597 if (rc) {
1598 DP_ERR(dev, "Problem copying data from user space\n");
1599 return rc;
1600 }
1601
1602
1603 rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr,
1604 ureq.sq_len, 0, 0, alloc_and_init);
1605 if (rc)
1606 return rc;
1607
1608 if (!qp->srq) {
1609
1610 rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr,
1611 ureq.rq_len, 0, 0, alloc_and_init);
1612 if (rc)
1613 return rc;
1614 }
1615
1616 memset(&in_params, 0, sizeof(in_params));
1617 qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1618 in_params.qp_handle_lo = ureq.qp_handle_lo;
1619 in_params.qp_handle_hi = ureq.qp_handle_hi;
1620 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1621 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1622 if (!qp->srq) {
1623 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1624 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1625 }
1626
1627 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1628 &in_params, &out_params);
1629
1630 if (!qp->qed_qp) {
1631 rc = -ENOMEM;
1632 goto err1;
1633 }
1634
1635 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1636 qedr_iwarp_populate_user_qp(dev, qp, &out_params);
1637
1638 qp->qp_id = out_params.qp_id;
1639 qp->icid = out_params.icid;
1640
1641 rc = qedr_copy_qp_uresp(dev, qp, udata);
1642 if (rc)
1643 goto err;
1644
1645 qedr_qp_user_print(dev, qp);
1646
1647 return 0;
1648err:
1649 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1650 if (rc)
1651 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1652
1653err1:
1654 qedr_cleanup_user(dev, qp);
1655 return rc;
1656}
1657
1658static void qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1659{
1660 qp->sq.db = dev->db_addr +
1661 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1662 qp->sq.db_data.data.icid = qp->icid;
1663
1664 qp->rq.db = dev->db_addr +
1665 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1666 qp->rq.db_data.data.icid = qp->icid;
1667 qp->rq.iwarp_db2 = dev->db_addr +
1668 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1669 qp->rq.iwarp_db2_data.data.icid = qp->icid;
1670 qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
1671}
1672
1673static int
1674qedr_roce_create_kernel_qp(struct qedr_dev *dev,
1675 struct qedr_qp *qp,
1676 struct qed_rdma_create_qp_in_params *in_params,
1677 u32 n_sq_elems, u32 n_rq_elems)
1678{
1679 struct qed_rdma_create_qp_out_params out_params;
1680 int rc;
1681
1682 rc = dev->ops->common->chain_alloc(dev->cdev,
1683 QED_CHAIN_USE_TO_PRODUCE,
1684 QED_CHAIN_MODE_PBL,
1685 QED_CHAIN_CNT_TYPE_U32,
1686 n_sq_elems,
1687 QEDR_SQE_ELEMENT_SIZE,
1688 &qp->sq.pbl, NULL);
1689
1690 if (rc)
1691 return rc;
1692
1693 in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
1694 in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
1695
1696 rc = dev->ops->common->chain_alloc(dev->cdev,
1697 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1698 QED_CHAIN_MODE_PBL,
1699 QED_CHAIN_CNT_TYPE_U32,
1700 n_rq_elems,
1701 QEDR_RQE_ELEMENT_SIZE,
1702 &qp->rq.pbl, NULL);
1703 if (rc)
1704 return rc;
1705
1706 in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
1707 in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
1708
1709 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1710 in_params, &out_params);
1711
1712 if (!qp->qed_qp)
1713 return -EINVAL;
1714
1715 qp->qp_id = out_params.qp_id;
1716 qp->icid = out_params.icid;
1717
1718 qedr_set_roce_db_info(dev, qp);
1719 return rc;
1720}
1721
1722static int
1723qedr_iwarp_create_kernel_qp(struct qedr_dev *dev,
1724 struct qedr_qp *qp,
1725 struct qed_rdma_create_qp_in_params *in_params,
1726 u32 n_sq_elems, u32 n_rq_elems)
1727{
1728 struct qed_rdma_create_qp_out_params out_params;
1729 struct qed_chain_ext_pbl ext_pbl;
1730 int rc;
1731
1732 in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems,
1733 QEDR_SQE_ELEMENT_SIZE,
1734 QED_CHAIN_MODE_PBL);
1735 in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems,
1736 QEDR_RQE_ELEMENT_SIZE,
1737 QED_CHAIN_MODE_PBL);
1738
1739 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1740 in_params, &out_params);
1741
1742 if (!qp->qed_qp)
1743 return -EINVAL;
1744
1745
1746 ext_pbl.p_pbl_virt = out_params.sq_pbl_virt;
1747 ext_pbl.p_pbl_phys = out_params.sq_pbl_phys;
1748
1749 rc = dev->ops->common->chain_alloc(dev->cdev,
1750 QED_CHAIN_USE_TO_PRODUCE,
1751 QED_CHAIN_MODE_PBL,
1752 QED_CHAIN_CNT_TYPE_U32,
1753 n_sq_elems,
1754 QEDR_SQE_ELEMENT_SIZE,
1755 &qp->sq.pbl, &ext_pbl);
1756
1757 if (rc)
1758 goto err;
1759
1760 ext_pbl.p_pbl_virt = out_params.rq_pbl_virt;
1761 ext_pbl.p_pbl_phys = out_params.rq_pbl_phys;
1762
1763 rc = dev->ops->common->chain_alloc(dev->cdev,
1764 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1765 QED_CHAIN_MODE_PBL,
1766 QED_CHAIN_CNT_TYPE_U32,
1767 n_rq_elems,
1768 QEDR_RQE_ELEMENT_SIZE,
1769 &qp->rq.pbl, &ext_pbl);
1770
1771 if (rc)
1772 goto err;
1773
1774 qp->qp_id = out_params.qp_id;
1775 qp->icid = out_params.icid;
1776
1777 qedr_set_iwarp_db_info(dev, qp);
1778 return rc;
1779
1780err:
1781 dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1782
1783 return rc;
1784}
1785
1786static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
1787{
1788 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
1789 kfree(qp->wqe_wr_id);
1790
1791 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
1792 kfree(qp->rqe_wr_id);
1793}
1794
1795static int qedr_create_kernel_qp(struct qedr_dev *dev,
1796 struct qedr_qp *qp,
1797 struct ib_pd *ibpd,
1798 struct ib_qp_init_attr *attrs)
1799{
1800 struct qed_rdma_create_qp_in_params in_params;
1801 struct qedr_pd *pd = get_qedr_pd(ibpd);
1802 int rc = -EINVAL;
1803 u32 n_rq_elems;
1804 u32 n_sq_elems;
1805 u32 n_sq_entries;
1806
1807 memset(&in_params, 0, sizeof(in_params));
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
1820 dev->attr.max_sqe);
1821
1822 qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id),
1823 GFP_KERNEL);
1824 if (!qp->wqe_wr_id) {
1825 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
1826 return -ENOMEM;
1827 }
1828
1829
1830 in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
1831 in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
1832
1833
1834
1835
1836
1837 qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
1838
1839
1840 qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id),
1841 GFP_KERNEL);
1842 if (!qp->rqe_wr_id) {
1843 DP_ERR(dev,
1844 "create qp: failed RQ shadow memory allocation\n");
1845 kfree(qp->wqe_wr_id);
1846 return -ENOMEM;
1847 }
1848
1849 qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
1850
1851 n_sq_entries = attrs->cap.max_send_wr;
1852 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
1853 n_sq_entries = max_t(u32, n_sq_entries, 1);
1854 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
1855
1856 n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
1857
1858 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1859 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params,
1860 n_sq_elems, n_rq_elems);
1861 else
1862 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
1863 n_sq_elems, n_rq_elems);
1864 if (rc)
1865 qedr_cleanup_kernel(dev, qp);
1866
1867 return rc;
1868}
1869
1870struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
1871 struct ib_qp_init_attr *attrs,
1872 struct ib_udata *udata)
1873{
1874 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
1875 struct qedr_pd *pd = get_qedr_pd(ibpd);
1876 struct qedr_qp *qp;
1877 struct ib_qp *ibqp;
1878 int rc = 0;
1879
1880 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
1881 udata ? "user library" : "kernel", pd);
1882
1883 rc = qedr_check_qp_attrs(ibpd, dev, attrs, udata);
1884 if (rc)
1885 return ERR_PTR(rc);
1886
1887 DP_DEBUG(dev, QEDR_MSG_QP,
1888 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
1889 udata ? "user library" : "kernel", attrs->event_handler, pd,
1890 get_qedr_cq(attrs->send_cq),
1891 get_qedr_cq(attrs->send_cq)->icid,
1892 get_qedr_cq(attrs->recv_cq),
1893 attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0);
1894
1895 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1896 if (!qp) {
1897 DP_ERR(dev, "create qp: failed allocating memory\n");
1898 return ERR_PTR(-ENOMEM);
1899 }
1900
1901 qedr_set_common_qp_params(dev, qp, pd, attrs);
1902
1903 if (attrs->qp_type == IB_QPT_GSI) {
1904 ibqp = qedr_create_gsi_qp(dev, attrs, qp);
1905 if (IS_ERR(ibqp))
1906 kfree(qp);
1907 return ibqp;
1908 }
1909
1910 if (udata)
1911 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
1912 else
1913 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
1914
1915 if (rc)
1916 goto err;
1917
1918 qp->ibqp.qp_num = qp->qp_id;
1919
1920 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1921 rc = xa_insert_irq(&dev->qps, qp->qp_id, qp, GFP_KERNEL);
1922 if (rc)
1923 goto err;
1924 }
1925
1926 return &qp->ibqp;
1927
1928err:
1929 kfree(qp);
1930
1931 return ERR_PTR(-EFAULT);
1932}
1933
1934static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
1935{
1936 switch (qp_state) {
1937 case QED_ROCE_QP_STATE_RESET:
1938 return IB_QPS_RESET;
1939 case QED_ROCE_QP_STATE_INIT:
1940 return IB_QPS_INIT;
1941 case QED_ROCE_QP_STATE_RTR:
1942 return IB_QPS_RTR;
1943 case QED_ROCE_QP_STATE_RTS:
1944 return IB_QPS_RTS;
1945 case QED_ROCE_QP_STATE_SQD:
1946 return IB_QPS_SQD;
1947 case QED_ROCE_QP_STATE_ERR:
1948 return IB_QPS_ERR;
1949 case QED_ROCE_QP_STATE_SQE:
1950 return IB_QPS_SQE;
1951 }
1952 return IB_QPS_ERR;
1953}
1954
1955static enum qed_roce_qp_state qedr_get_state_from_ibqp(
1956 enum ib_qp_state qp_state)
1957{
1958 switch (qp_state) {
1959 case IB_QPS_RESET:
1960 return QED_ROCE_QP_STATE_RESET;
1961 case IB_QPS_INIT:
1962 return QED_ROCE_QP_STATE_INIT;
1963 case IB_QPS_RTR:
1964 return QED_ROCE_QP_STATE_RTR;
1965 case IB_QPS_RTS:
1966 return QED_ROCE_QP_STATE_RTS;
1967 case IB_QPS_SQD:
1968 return QED_ROCE_QP_STATE_SQD;
1969 case IB_QPS_ERR:
1970 return QED_ROCE_QP_STATE_ERR;
1971 default:
1972 return QED_ROCE_QP_STATE_ERR;
1973 }
1974}
1975
1976static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
1977{
1978 qed_chain_reset(&qph->pbl);
1979 qph->prod = 0;
1980 qph->cons = 0;
1981 qph->wqe_cons = 0;
1982 qph->db_data.data.value = cpu_to_le16(0);
1983}
1984
1985static int qedr_update_qp_state(struct qedr_dev *dev,
1986 struct qedr_qp *qp,
1987 enum qed_roce_qp_state cur_state,
1988 enum qed_roce_qp_state new_state)
1989{
1990 int status = 0;
1991
1992 if (new_state == cur_state)
1993 return 0;
1994
1995 switch (cur_state) {
1996 case QED_ROCE_QP_STATE_RESET:
1997 switch (new_state) {
1998 case QED_ROCE_QP_STATE_INIT:
1999 qp->prev_wqe_size = 0;
2000 qedr_reset_qp_hwq_info(&qp->sq);
2001 qedr_reset_qp_hwq_info(&qp->rq);
2002 break;
2003 default:
2004 status = -EINVAL;
2005 break;
2006 }
2007 break;
2008 case QED_ROCE_QP_STATE_INIT:
2009 switch (new_state) {
2010 case QED_ROCE_QP_STATE_RTR:
2011
2012
2013
2014
2015 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2016 writel(qp->rq.db_data.raw, qp->rq.db);
2017 }
2018 break;
2019 case QED_ROCE_QP_STATE_ERR:
2020 break;
2021 default:
2022
2023 status = -EINVAL;
2024 break;
2025 }
2026 break;
2027 case QED_ROCE_QP_STATE_RTR:
2028
2029 switch (new_state) {
2030 case QED_ROCE_QP_STATE_RTS:
2031 break;
2032 case QED_ROCE_QP_STATE_ERR:
2033 break;
2034 default:
2035
2036 status = -EINVAL;
2037 break;
2038 }
2039 break;
2040 case QED_ROCE_QP_STATE_RTS:
2041
2042 switch (new_state) {
2043 case QED_ROCE_QP_STATE_SQD:
2044 break;
2045 case QED_ROCE_QP_STATE_ERR:
2046 break;
2047 default:
2048
2049 status = -EINVAL;
2050 break;
2051 }
2052 break;
2053 case QED_ROCE_QP_STATE_SQD:
2054
2055 switch (new_state) {
2056 case QED_ROCE_QP_STATE_RTS:
2057 case QED_ROCE_QP_STATE_ERR:
2058 break;
2059 default:
2060
2061 status = -EINVAL;
2062 break;
2063 }
2064 break;
2065 case QED_ROCE_QP_STATE_ERR:
2066
2067 switch (new_state) {
2068 case QED_ROCE_QP_STATE_RESET:
2069 if ((qp->rq.prod != qp->rq.cons) ||
2070 (qp->sq.prod != qp->sq.cons)) {
2071 DP_NOTICE(dev,
2072 "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
2073 qp->rq.prod, qp->rq.cons, qp->sq.prod,
2074 qp->sq.cons);
2075 status = -EINVAL;
2076 }
2077 break;
2078 default:
2079 status = -EINVAL;
2080 break;
2081 }
2082 break;
2083 default:
2084 status = -EINVAL;
2085 break;
2086 }
2087
2088 return status;
2089}
2090
2091int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2092 int attr_mask, struct ib_udata *udata)
2093{
2094 struct qedr_qp *qp = get_qedr_qp(ibqp);
2095 struct qed_rdma_modify_qp_in_params qp_params = { 0 };
2096 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
2097 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
2098 enum ib_qp_state old_qp_state, new_qp_state;
2099 enum qed_roce_qp_state cur_state;
2100 int rc = 0;
2101
2102 DP_DEBUG(dev, QEDR_MSG_QP,
2103 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
2104 attr->qp_state);
2105
2106 old_qp_state = qedr_get_ibqp_state(qp->state);
2107 if (attr_mask & IB_QP_STATE)
2108 new_qp_state = attr->qp_state;
2109 else
2110 new_qp_state = old_qp_state;
2111
2112 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2113 if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state,
2114 ibqp->qp_type, attr_mask)) {
2115 DP_ERR(dev,
2116 "modify qp: invalid attribute mask=0x%x specified for\n"
2117 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
2118 attr_mask, qp->qp_id, ibqp->qp_type,
2119 old_qp_state, new_qp_state);
2120 rc = -EINVAL;
2121 goto err;
2122 }
2123 }
2124
2125
2126 if (attr_mask & IB_QP_STATE) {
2127 SET_FIELD(qp_params.modify_flags,
2128 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
2129 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
2130 }
2131
2132 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
2133 qp_params.sqd_async = true;
2134
2135 if (attr_mask & IB_QP_PKEY_INDEX) {
2136 SET_FIELD(qp_params.modify_flags,
2137 QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
2138 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
2139 rc = -EINVAL;
2140 goto err;
2141 }
2142
2143 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
2144 }
2145
2146 if (attr_mask & IB_QP_QKEY)
2147 qp->qkey = attr->qkey;
2148
2149 if (attr_mask & IB_QP_ACCESS_FLAGS) {
2150 SET_FIELD(qp_params.modify_flags,
2151 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
2152 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
2153 IB_ACCESS_REMOTE_READ;
2154 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
2155 IB_ACCESS_REMOTE_WRITE;
2156 qp_params.incoming_atomic_en = attr->qp_access_flags &
2157 IB_ACCESS_REMOTE_ATOMIC;
2158 }
2159
2160 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
2161 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2162 return -EINVAL;
2163
2164 if (attr_mask & IB_QP_PATH_MTU) {
2165 if (attr->path_mtu < IB_MTU_256 ||
2166 attr->path_mtu > IB_MTU_4096) {
2167 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
2168 rc = -EINVAL;
2169 goto err;
2170 }
2171 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
2172 ib_mtu_enum_to_int(iboe_get_mtu
2173 (dev->ndev->mtu)));
2174 }
2175
2176 if (!qp->mtu) {
2177 qp->mtu =
2178 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2179 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
2180 }
2181
2182 SET_FIELD(qp_params.modify_flags,
2183 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
2184
2185 qp_params.traffic_class_tos = grh->traffic_class;
2186 qp_params.flow_label = grh->flow_label;
2187 qp_params.hop_limit_ttl = grh->hop_limit;
2188
2189 qp->sgid_idx = grh->sgid_index;
2190
2191 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
2192 if (rc) {
2193 DP_ERR(dev,
2194 "modify qp: problems with GID index %d (rc=%d)\n",
2195 grh->sgid_index, rc);
2196 return rc;
2197 }
2198
2199 rc = qedr_get_dmac(dev, &attr->ah_attr,
2200 qp_params.remote_mac_addr);
2201 if (rc)
2202 return rc;
2203
2204 qp_params.use_local_mac = true;
2205 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
2206
2207 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
2208 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
2209 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
2210 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
2211 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
2212 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
2213 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
2214 qp_params.remote_mac_addr);
2215
2216 qp_params.mtu = qp->mtu;
2217 qp_params.lb_indication = false;
2218 }
2219
2220 if (!qp_params.mtu) {
2221
2222 if (qp->mtu)
2223 qp_params.mtu = qp->mtu;
2224 else
2225 qp_params.mtu =
2226 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2227 }
2228
2229 if (attr_mask & IB_QP_TIMEOUT) {
2230 SET_FIELD(qp_params.modify_flags,
2231 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243 if (attr->timeout)
2244 qp_params.ack_timeout =
2245 1 << max_t(int, attr->timeout - 8, 0);
2246 else
2247 qp_params.ack_timeout = 0;
2248 }
2249
2250 if (attr_mask & IB_QP_RETRY_CNT) {
2251 SET_FIELD(qp_params.modify_flags,
2252 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
2253 qp_params.retry_cnt = attr->retry_cnt;
2254 }
2255
2256 if (attr_mask & IB_QP_RNR_RETRY) {
2257 SET_FIELD(qp_params.modify_flags,
2258 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
2259 qp_params.rnr_retry_cnt = attr->rnr_retry;
2260 }
2261
2262 if (attr_mask & IB_QP_RQ_PSN) {
2263 SET_FIELD(qp_params.modify_flags,
2264 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
2265 qp_params.rq_psn = attr->rq_psn;
2266 qp->rq_psn = attr->rq_psn;
2267 }
2268
2269 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2270 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
2271 rc = -EINVAL;
2272 DP_ERR(dev,
2273 "unsupported max_rd_atomic=%d, supported=%d\n",
2274 attr->max_rd_atomic,
2275 dev->attr.max_qp_req_rd_atomic_resc);
2276 goto err;
2277 }
2278
2279 SET_FIELD(qp_params.modify_flags,
2280 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
2281 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
2282 }
2283
2284 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
2285 SET_FIELD(qp_params.modify_flags,
2286 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
2287 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
2288 }
2289
2290 if (attr_mask & IB_QP_SQ_PSN) {
2291 SET_FIELD(qp_params.modify_flags,
2292 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
2293 qp_params.sq_psn = attr->sq_psn;
2294 qp->sq_psn = attr->sq_psn;
2295 }
2296
2297 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2298 if (attr->max_dest_rd_atomic >
2299 dev->attr.max_qp_resp_rd_atomic_resc) {
2300 DP_ERR(dev,
2301 "unsupported max_dest_rd_atomic=%d, supported=%d\n",
2302 attr->max_dest_rd_atomic,
2303 dev->attr.max_qp_resp_rd_atomic_resc);
2304
2305 rc = -EINVAL;
2306 goto err;
2307 }
2308
2309 SET_FIELD(qp_params.modify_flags,
2310 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
2311 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
2312 }
2313
2314 if (attr_mask & IB_QP_DEST_QPN) {
2315 SET_FIELD(qp_params.modify_flags,
2316 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
2317
2318 qp_params.dest_qp = attr->dest_qp_num;
2319 qp->dest_qp_num = attr->dest_qp_num;
2320 }
2321
2322 cur_state = qp->state;
2323
2324
2325
2326
2327
2328
2329 if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI &&
2330 !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR)
2331 qp->state = QED_ROCE_QP_STATE_ERR;
2332
2333 if (qp->qp_type != IB_QPT_GSI)
2334 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
2335 qp->qed_qp, &qp_params);
2336
2337 if (attr_mask & IB_QP_STATE) {
2338 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
2339 rc = qedr_update_qp_state(dev, qp, cur_state,
2340 qp_params.new_state);
2341 qp->state = qp_params.new_state;
2342 }
2343
2344err:
2345 return rc;
2346}
2347
2348static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
2349{
2350 int ib_qp_acc_flags = 0;
2351
2352 if (params->incoming_rdma_write_en)
2353 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
2354 if (params->incoming_rdma_read_en)
2355 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2356 if (params->incoming_atomic_en)
2357 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2358 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2359 return ib_qp_acc_flags;
2360}
2361
2362int qedr_query_qp(struct ib_qp *ibqp,
2363 struct ib_qp_attr *qp_attr,
2364 int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2365{
2366 struct qed_rdma_query_qp_out_params params;
2367 struct qedr_qp *qp = get_qedr_qp(ibqp);
2368 struct qedr_dev *dev = qp->dev;
2369 int rc = 0;
2370
2371 memset(¶ms, 0, sizeof(params));
2372
2373 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, ¶ms);
2374 if (rc)
2375 goto err;
2376
2377 memset(qp_attr, 0, sizeof(*qp_attr));
2378 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2379
2380 qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2381 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
2382 qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
2383 qp_attr->path_mig_state = IB_MIG_MIGRATED;
2384 qp_attr->rq_psn = params.rq_psn;
2385 qp_attr->sq_psn = params.sq_psn;
2386 qp_attr->dest_qp_num = params.dest_qp;
2387
2388 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(¶ms);
2389
2390 qp_attr->cap.max_send_wr = qp->sq.max_wr;
2391 qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2392 qp_attr->cap.max_send_sge = qp->sq.max_sges;
2393 qp_attr->cap.max_recv_sge = qp->rq.max_sges;
2394 qp_attr->cap.max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE;
2395 qp_init_attr->cap = qp_attr->cap;
2396
2397 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2398 rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
2399 params.flow_label, qp->sgid_idx,
2400 params.hop_limit_ttl, params.traffic_class_tos);
2401 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, ¶ms.dgid.bytes[0]);
2402 rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
2403 rdma_ah_set_sl(&qp_attr->ah_attr, 0);
2404 qp_attr->timeout = params.timeout;
2405 qp_attr->rnr_retry = params.rnr_retry;
2406 qp_attr->retry_cnt = params.retry_cnt;
2407 qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2408 qp_attr->pkey_index = params.pkey_index;
2409 qp_attr->port_num = 1;
2410 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
2411 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
2412 qp_attr->alt_pkey_index = 0;
2413 qp_attr->alt_port_num = 0;
2414 qp_attr->alt_timeout = 0;
2415 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2416
2417 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2418 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2419 qp_attr->max_rd_atomic = params.max_rd_atomic;
2420 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2421
2422 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2423 qp_attr->cap.max_inline_data);
2424
2425err:
2426 return rc;
2427}
2428
2429static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp,
2430 struct ib_udata *udata)
2431{
2432 int rc = 0;
2433
2434 if (qp->qp_type != IB_QPT_GSI) {
2435 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2436 if (rc)
2437 return rc;
2438 }
2439
2440 if (udata)
2441 qedr_cleanup_user(dev, qp);
2442 else
2443 qedr_cleanup_kernel(dev, qp);
2444
2445 return 0;
2446}
2447
2448int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
2449{
2450 struct qedr_qp *qp = get_qedr_qp(ibqp);
2451 struct qedr_dev *dev = qp->dev;
2452 struct ib_qp_attr attr;
2453 int attr_mask = 0;
2454 int rc = 0;
2455
2456 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2457 qp, qp->qp_type);
2458
2459 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2460 if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2461 (qp->state != QED_ROCE_QP_STATE_ERR) &&
2462 (qp->state != QED_ROCE_QP_STATE_INIT)) {
2463
2464 attr.qp_state = IB_QPS_ERR;
2465 attr_mask |= IB_QP_STATE;
2466
2467
2468 qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2469 }
2470 } else {
2471
2472 if (qp->ep) {
2473 int wait_count = 1;
2474
2475 while (qp->ep->during_connect) {
2476 DP_DEBUG(dev, QEDR_MSG_QP,
2477 "Still in during connect/accept\n");
2478
2479 msleep(100);
2480 if (wait_count++ > 200) {
2481 DP_NOTICE(dev,
2482 "during connect timeout\n");
2483 break;
2484 }
2485 }
2486 }
2487 }
2488
2489 if (qp->qp_type == IB_QPT_GSI)
2490 qedr_destroy_gsi_qp(dev);
2491
2492 qedr_free_qp_resources(dev, qp, udata);
2493
2494 if (atomic_dec_and_test(&qp->refcnt) &&
2495 rdma_protocol_iwarp(&dev->ibdev, 1)) {
2496 xa_erase_irq(&dev->qps, qp->qp_id);
2497 kfree(qp);
2498 }
2499 return rc;
2500}
2501
2502int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_attr *attr, u32 flags,
2503 struct ib_udata *udata)
2504{
2505 struct qedr_ah *ah = get_qedr_ah(ibah);
2506
2507 rdma_copy_ah_attr(&ah->attr, attr);
2508
2509 return 0;
2510}
2511
2512void qedr_destroy_ah(struct ib_ah *ibah, u32 flags)
2513{
2514 struct qedr_ah *ah = get_qedr_ah(ibah);
2515
2516 rdma_destroy_ah_attr(&ah->attr);
2517}
2518
2519static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2520{
2521 struct qedr_pbl *pbl, *tmp;
2522
2523 if (info->pbl_table)
2524 list_add_tail(&info->pbl_table->list_entry,
2525 &info->free_pbl_list);
2526
2527 if (!list_empty(&info->inuse_pbl_list))
2528 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2529
2530 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2531 list_del(&pbl->list_entry);
2532 qedr_free_pbl(dev, &info->pbl_info, pbl);
2533 }
2534}
2535
2536static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2537 size_t page_list_len, bool two_layered)
2538{
2539 struct qedr_pbl *tmp;
2540 int rc;
2541
2542 INIT_LIST_HEAD(&info->free_pbl_list);
2543 INIT_LIST_HEAD(&info->inuse_pbl_list);
2544
2545 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2546 page_list_len, two_layered);
2547 if (rc)
2548 goto done;
2549
2550 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2551 if (IS_ERR(info->pbl_table)) {
2552 rc = PTR_ERR(info->pbl_table);
2553 goto done;
2554 }
2555
2556 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2557 &info->pbl_table->pa);
2558
2559
2560
2561
2562 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2563 if (IS_ERR(tmp)) {
2564 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2565 goto done;
2566 }
2567
2568 list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2569
2570 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2571
2572done:
2573 if (rc)
2574 free_mr_info(dev, info);
2575
2576 return rc;
2577}
2578
2579struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2580 u64 usr_addr, int acc, struct ib_udata *udata)
2581{
2582 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2583 struct qedr_mr *mr;
2584 struct qedr_pd *pd;
2585 int rc = -ENOMEM;
2586
2587 pd = get_qedr_pd(ibpd);
2588 DP_DEBUG(dev, QEDR_MSG_MR,
2589 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2590 pd->pd_id, start, len, usr_addr, acc);
2591
2592 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2593 return ERR_PTR(-EINVAL);
2594
2595 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2596 if (!mr)
2597 return ERR_PTR(rc);
2598
2599 mr->type = QEDR_MR_USER;
2600
2601 mr->umem = ib_umem_get(udata, start, len, acc, 0);
2602 if (IS_ERR(mr->umem)) {
2603 rc = -EFAULT;
2604 goto err0;
2605 }
2606
2607 rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);
2608 if (rc)
2609 goto err1;
2610
2611 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
2612 &mr->info.pbl_info, PAGE_SHIFT);
2613
2614 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2615 if (rc) {
2616 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2617 goto err1;
2618 }
2619
2620
2621 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2622 mr->hw_mr.key = 0;
2623 mr->hw_mr.pd = pd->pd_id;
2624 mr->hw_mr.local_read = 1;
2625 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2626 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2627 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2628 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2629 mr->hw_mr.mw_bind = false;
2630 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
2631 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2632 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2633 mr->hw_mr.page_size_log = PAGE_SHIFT;
2634 mr->hw_mr.fbo = ib_umem_offset(mr->umem);
2635 mr->hw_mr.length = len;
2636 mr->hw_mr.vaddr = usr_addr;
2637 mr->hw_mr.zbva = false;
2638 mr->hw_mr.phy_mr = false;
2639 mr->hw_mr.dma_mr = false;
2640
2641 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2642 if (rc) {
2643 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2644 goto err2;
2645 }
2646
2647 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2648 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2649 mr->hw_mr.remote_atomic)
2650 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2651
2652 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
2653 mr->ibmr.lkey);
2654 return &mr->ibmr;
2655
2656err2:
2657 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2658err1:
2659 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2660err0:
2661 kfree(mr);
2662 return ERR_PTR(rc);
2663}
2664
2665int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
2666{
2667 struct qedr_mr *mr = get_qedr_mr(ib_mr);
2668 struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
2669 int rc = 0;
2670
2671 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
2672 if (rc)
2673 return rc;
2674
2675 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2676
2677 if ((mr->type != QEDR_MR_DMA) && (mr->type != QEDR_MR_FRMR))
2678 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2679
2680
2681 ib_umem_release(mr->umem);
2682
2683 kfree(mr);
2684
2685 return rc;
2686}
2687
2688static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
2689 int max_page_list_len)
2690{
2691 struct qedr_pd *pd = get_qedr_pd(ibpd);
2692 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2693 struct qedr_mr *mr;
2694 int rc = -ENOMEM;
2695
2696 DP_DEBUG(dev, QEDR_MSG_MR,
2697 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
2698 max_page_list_len);
2699
2700 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2701 if (!mr)
2702 return ERR_PTR(rc);
2703
2704 mr->dev = dev;
2705 mr->type = QEDR_MR_FRMR;
2706
2707 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
2708 if (rc)
2709 goto err0;
2710
2711 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2712 if (rc) {
2713 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2714 goto err0;
2715 }
2716
2717
2718 mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
2719 mr->hw_mr.key = 0;
2720 mr->hw_mr.pd = pd->pd_id;
2721 mr->hw_mr.local_read = 1;
2722 mr->hw_mr.local_write = 0;
2723 mr->hw_mr.remote_read = 0;
2724 mr->hw_mr.remote_write = 0;
2725 mr->hw_mr.remote_atomic = 0;
2726 mr->hw_mr.mw_bind = false;
2727 mr->hw_mr.pbl_ptr = 0;
2728 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2729 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2730 mr->hw_mr.fbo = 0;
2731 mr->hw_mr.length = 0;
2732 mr->hw_mr.vaddr = 0;
2733 mr->hw_mr.zbva = false;
2734 mr->hw_mr.phy_mr = true;
2735 mr->hw_mr.dma_mr = false;
2736
2737 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2738 if (rc) {
2739 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2740 goto err1;
2741 }
2742
2743 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2744 mr->ibmr.rkey = mr->ibmr.lkey;
2745
2746 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
2747 return mr;
2748
2749err1:
2750 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2751err0:
2752 kfree(mr);
2753 return ERR_PTR(rc);
2754}
2755
2756struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
2757 u32 max_num_sg, struct ib_udata *udata)
2758{
2759 struct qedr_mr *mr;
2760
2761 if (mr_type != IB_MR_TYPE_MEM_REG)
2762 return ERR_PTR(-EINVAL);
2763
2764 mr = __qedr_alloc_mr(ibpd, max_num_sg);
2765
2766 if (IS_ERR(mr))
2767 return ERR_PTR(-EINVAL);
2768
2769 return &mr->ibmr;
2770}
2771
2772static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
2773{
2774 struct qedr_mr *mr = get_qedr_mr(ibmr);
2775 struct qedr_pbl *pbl_table;
2776 struct regpair *pbe;
2777 u32 pbes_in_page;
2778
2779 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
2780 DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages);
2781 return -ENOMEM;
2782 }
2783
2784 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
2785 mr->npages, addr);
2786
2787 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
2788 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
2789 pbe = (struct regpair *)pbl_table->va;
2790 pbe += mr->npages % pbes_in_page;
2791 pbe->lo = cpu_to_le32((u32)addr);
2792 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
2793
2794 mr->npages++;
2795
2796 return 0;
2797}
2798
2799static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
2800{
2801 int work = info->completed - info->completed_handled - 1;
2802
2803 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
2804 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
2805 struct qedr_pbl *pbl;
2806
2807
2808
2809
2810
2811
2812 pbl = list_first_entry(&info->inuse_pbl_list,
2813 struct qedr_pbl, list_entry);
2814 list_move_tail(&pbl->list_entry, &info->free_pbl_list);
2815 info->completed_handled++;
2816 }
2817}
2818
2819int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
2820 int sg_nents, unsigned int *sg_offset)
2821{
2822 struct qedr_mr *mr = get_qedr_mr(ibmr);
2823
2824 mr->npages = 0;
2825
2826 handle_completed_mrs(mr->dev, &mr->info);
2827 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
2828}
2829
2830struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
2831{
2832 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2833 struct qedr_pd *pd = get_qedr_pd(ibpd);
2834 struct qedr_mr *mr;
2835 int rc;
2836
2837 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2838 if (!mr)
2839 return ERR_PTR(-ENOMEM);
2840
2841 mr->type = QEDR_MR_DMA;
2842
2843 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2844 if (rc) {
2845 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2846 goto err1;
2847 }
2848
2849
2850 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2851 mr->hw_mr.pd = pd->pd_id;
2852 mr->hw_mr.local_read = 1;
2853 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2854 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2855 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2856 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2857 mr->hw_mr.dma_mr = true;
2858
2859 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2860 if (rc) {
2861 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2862 goto err2;
2863 }
2864
2865 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2866 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2867 mr->hw_mr.remote_atomic)
2868 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2869
2870 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
2871 return &mr->ibmr;
2872
2873err2:
2874 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2875err1:
2876 kfree(mr);
2877 return ERR_PTR(rc);
2878}
2879
2880static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
2881{
2882 return (((wq->prod + 1) % wq->max_wr) == wq->cons);
2883}
2884
2885static int sge_data_len(struct ib_sge *sg_list, int num_sge)
2886{
2887 int i, len = 0;
2888
2889 for (i = 0; i < num_sge; i++)
2890 len += sg_list[i].length;
2891
2892 return len;
2893}
2894
2895static void swap_wqe_data64(u64 *p)
2896{
2897 int i;
2898
2899 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
2900 *p = cpu_to_be64(cpu_to_le64(*p));
2901}
2902
2903static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
2904 struct qedr_qp *qp, u8 *wqe_size,
2905 const struct ib_send_wr *wr,
2906 const struct ib_send_wr **bad_wr,
2907 u8 *bits, u8 bit)
2908{
2909 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
2910 char *seg_prt, *wqe;
2911 int i, seg_siz;
2912
2913 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
2914 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
2915 *bad_wr = wr;
2916 return 0;
2917 }
2918
2919 if (!data_size)
2920 return data_size;
2921
2922 *bits |= bit;
2923
2924 seg_prt = NULL;
2925 wqe = NULL;
2926 seg_siz = 0;
2927
2928
2929 for (i = 0; i < wr->num_sge; i++) {
2930 u32 len = wr->sg_list[i].length;
2931 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
2932
2933 while (len > 0) {
2934 u32 cur;
2935
2936
2937 if (!seg_siz) {
2938 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
2939 seg_prt = wqe;
2940 seg_siz = sizeof(struct rdma_sq_common_wqe);
2941 (*wqe_size)++;
2942 }
2943
2944
2945 cur = min_t(u32, len, seg_siz);
2946 memcpy(seg_prt, src, cur);
2947
2948
2949 seg_prt += cur;
2950 seg_siz -= cur;
2951
2952
2953 src += cur;
2954 len -= cur;
2955
2956
2957 if (!seg_siz)
2958 swap_wqe_data64((u64 *)wqe);
2959 }
2960 }
2961
2962
2963 if (seg_siz)
2964 swap_wqe_data64((u64 *)wqe);
2965
2966 return data_size;
2967}
2968
2969#define RQ_SGE_SET(sge, vaddr, vlength, vflags) \
2970 do { \
2971 DMA_REGPAIR_LE(sge->addr, vaddr); \
2972 (sge)->length = cpu_to_le32(vlength); \
2973 (sge)->flags = cpu_to_le32(vflags); \
2974 } while (0)
2975
2976#define SRQ_HDR_SET(hdr, vwr_id, num_sge) \
2977 do { \
2978 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \
2979 (hdr)->num_sges = num_sge; \
2980 } while (0)
2981
2982#define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \
2983 do { \
2984 DMA_REGPAIR_LE(sge->addr, vaddr); \
2985 (sge)->length = cpu_to_le32(vlength); \
2986 (sge)->l_key = cpu_to_le32(vlkey); \
2987 } while (0)
2988
2989static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
2990 const struct ib_send_wr *wr)
2991{
2992 u32 data_size = 0;
2993 int i;
2994
2995 for (i = 0; i < wr->num_sge; i++) {
2996 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
2997
2998 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
2999 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
3000 sge->length = cpu_to_le32(wr->sg_list[i].length);
3001 data_size += wr->sg_list[i].length;
3002 }
3003
3004 if (wqe_size)
3005 *wqe_size += wr->num_sge;
3006
3007 return data_size;
3008}
3009
3010static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
3011 struct qedr_qp *qp,
3012 struct rdma_sq_rdma_wqe_1st *rwqe,
3013 struct rdma_sq_rdma_wqe_2nd *rwqe2,
3014 const struct ib_send_wr *wr,
3015 const struct ib_send_wr **bad_wr)
3016{
3017 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
3018 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
3019
3020 if (wr->send_flags & IB_SEND_INLINE &&
3021 (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
3022 wr->opcode == IB_WR_RDMA_WRITE)) {
3023 u8 flags = 0;
3024
3025 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
3026 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
3027 bad_wr, &rwqe->flags, flags);
3028 }
3029
3030 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
3031}
3032
3033static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
3034 struct qedr_qp *qp,
3035 struct rdma_sq_send_wqe_1st *swqe,
3036 struct rdma_sq_send_wqe_2st *swqe2,
3037 const struct ib_send_wr *wr,
3038 const struct ib_send_wr **bad_wr)
3039{
3040 memset(swqe2, 0, sizeof(*swqe2));
3041 if (wr->send_flags & IB_SEND_INLINE) {
3042 u8 flags = 0;
3043
3044 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
3045 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
3046 bad_wr, &swqe->flags, flags);
3047 }
3048
3049 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
3050}
3051
3052static int qedr_prepare_reg(struct qedr_qp *qp,
3053 struct rdma_sq_fmr_wqe_1st *fwqe1,
3054 const struct ib_reg_wr *wr)
3055{
3056 struct qedr_mr *mr = get_qedr_mr(wr->mr);
3057 struct rdma_sq_fmr_wqe_2nd *fwqe2;
3058
3059 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
3060 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
3061 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
3062 fwqe1->l_key = wr->key;
3063
3064 fwqe2->access_ctrl = 0;
3065
3066 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
3067 !!(wr->access & IB_ACCESS_REMOTE_READ));
3068 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
3069 !!(wr->access & IB_ACCESS_REMOTE_WRITE));
3070 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
3071 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
3072 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
3073 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
3074 !!(wr->access & IB_ACCESS_LOCAL_WRITE));
3075 fwqe2->fmr_ctrl = 0;
3076
3077 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
3078 ilog2(mr->ibmr.page_size) - 12);
3079
3080 fwqe2->length_hi = 0;
3081 fwqe2->length_lo = mr->ibmr.length;
3082 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
3083 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
3084
3085 qp->wqe_wr_id[qp->sq.prod].mr = mr;
3086
3087 return 0;
3088}
3089
3090static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
3091{
3092 switch (opcode) {
3093 case IB_WR_RDMA_WRITE:
3094 case IB_WR_RDMA_WRITE_WITH_IMM:
3095 return IB_WC_RDMA_WRITE;
3096 case IB_WR_SEND_WITH_IMM:
3097 case IB_WR_SEND:
3098 case IB_WR_SEND_WITH_INV:
3099 return IB_WC_SEND;
3100 case IB_WR_RDMA_READ:
3101 case IB_WR_RDMA_READ_WITH_INV:
3102 return IB_WC_RDMA_READ;
3103 case IB_WR_ATOMIC_CMP_AND_SWP:
3104 return IB_WC_COMP_SWAP;
3105 case IB_WR_ATOMIC_FETCH_AND_ADD:
3106 return IB_WC_FETCH_ADD;
3107 case IB_WR_REG_MR:
3108 return IB_WC_REG_MR;
3109 case IB_WR_LOCAL_INV:
3110 return IB_WC_LOCAL_INV;
3111 default:
3112 return IB_WC_SEND;
3113 }
3114}
3115
3116static inline bool qedr_can_post_send(struct qedr_qp *qp,
3117 const struct ib_send_wr *wr)
3118{
3119 int wq_is_full, err_wr, pbl_is_full;
3120 struct qedr_dev *dev = qp->dev;
3121
3122
3123 err_wr = wr->num_sge > qp->sq.max_sges;
3124 wq_is_full = qedr_wq_is_full(&qp->sq);
3125 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
3126 QEDR_MAX_SQE_ELEMENTS_PER_SQE;
3127 if (wq_is_full || err_wr || pbl_is_full) {
3128 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
3129 DP_ERR(dev,
3130 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
3131 qp);
3132 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
3133 }
3134
3135 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
3136 DP_ERR(dev,
3137 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
3138 qp);
3139 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
3140 }
3141
3142 if (pbl_is_full &&
3143 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
3144 DP_ERR(dev,
3145 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
3146 qp);
3147 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
3148 }
3149 return false;
3150 }
3151 return true;
3152}
3153
3154static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3155 const struct ib_send_wr **bad_wr)
3156{
3157 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3158 struct qedr_qp *qp = get_qedr_qp(ibqp);
3159 struct rdma_sq_atomic_wqe_1st *awqe1;
3160 struct rdma_sq_atomic_wqe_2nd *awqe2;
3161 struct rdma_sq_atomic_wqe_3rd *awqe3;
3162 struct rdma_sq_send_wqe_2st *swqe2;
3163 struct rdma_sq_local_inv_wqe *iwqe;
3164 struct rdma_sq_rdma_wqe_2nd *rwqe2;
3165 struct rdma_sq_send_wqe_1st *swqe;
3166 struct rdma_sq_rdma_wqe_1st *rwqe;
3167 struct rdma_sq_fmr_wqe_1st *fwqe1;
3168 struct rdma_sq_common_wqe *wqe;
3169 u32 length;
3170 int rc = 0;
3171 bool comp;
3172
3173 if (!qedr_can_post_send(qp, wr)) {
3174 *bad_wr = wr;
3175 return -ENOMEM;
3176 }
3177
3178 wqe = qed_chain_produce(&qp->sq.pbl);
3179 qp->wqe_wr_id[qp->sq.prod].signaled =
3180 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
3181
3182 wqe->flags = 0;
3183 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
3184 !!(wr->send_flags & IB_SEND_SOLICITED));
3185 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
3186 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
3187 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
3188 !!(wr->send_flags & IB_SEND_FENCE));
3189 wqe->prev_wqe_size = qp->prev_wqe_size;
3190
3191 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
3192
3193 switch (wr->opcode) {
3194 case IB_WR_SEND_WITH_IMM:
3195 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3196 rc = -EINVAL;
3197 *bad_wr = wr;
3198 break;
3199 }
3200 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
3201 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3202 swqe->wqe_size = 2;
3203 swqe2 = qed_chain_produce(&qp->sq.pbl);
3204
3205 swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
3206 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3207 wr, bad_wr);
3208 swqe->length = cpu_to_le32(length);
3209 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3210 qp->prev_wqe_size = swqe->wqe_size;
3211 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3212 break;
3213 case IB_WR_SEND:
3214 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
3215 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3216
3217 swqe->wqe_size = 2;
3218 swqe2 = qed_chain_produce(&qp->sq.pbl);
3219 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3220 wr, bad_wr);
3221 swqe->length = cpu_to_le32(length);
3222 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3223 qp->prev_wqe_size = swqe->wqe_size;
3224 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3225 break;
3226 case IB_WR_SEND_WITH_INV:
3227 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
3228 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3229 swqe2 = qed_chain_produce(&qp->sq.pbl);
3230 swqe->wqe_size = 2;
3231 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
3232 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3233 wr, bad_wr);
3234 swqe->length = cpu_to_le32(length);
3235 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3236 qp->prev_wqe_size = swqe->wqe_size;
3237 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3238 break;
3239
3240 case IB_WR_RDMA_WRITE_WITH_IMM:
3241 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3242 rc = -EINVAL;
3243 *bad_wr = wr;
3244 break;
3245 }
3246 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
3247 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3248
3249 rwqe->wqe_size = 2;
3250 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
3251 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3252 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3253 wr, bad_wr);
3254 rwqe->length = cpu_to_le32(length);
3255 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3256 qp->prev_wqe_size = rwqe->wqe_size;
3257 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3258 break;
3259 case IB_WR_RDMA_WRITE:
3260 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
3261 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3262
3263 rwqe->wqe_size = 2;
3264 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3265 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3266 wr, bad_wr);
3267 rwqe->length = cpu_to_le32(length);
3268 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3269 qp->prev_wqe_size = rwqe->wqe_size;
3270 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3271 break;
3272 case IB_WR_RDMA_READ_WITH_INV:
3273 SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1);
3274
3275
3276 case IB_WR_RDMA_READ:
3277 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
3278 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3279
3280 rwqe->wqe_size = 2;
3281 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3282 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3283 wr, bad_wr);
3284 rwqe->length = cpu_to_le32(length);
3285 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3286 qp->prev_wqe_size = rwqe->wqe_size;
3287 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3288 break;
3289
3290 case IB_WR_ATOMIC_CMP_AND_SWP:
3291 case IB_WR_ATOMIC_FETCH_AND_ADD:
3292 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
3293 awqe1->wqe_size = 4;
3294
3295 awqe2 = qed_chain_produce(&qp->sq.pbl);
3296 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
3297 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
3298
3299 awqe3 = qed_chain_produce(&qp->sq.pbl);
3300
3301 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
3302 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
3303 DMA_REGPAIR_LE(awqe3->swap_data,
3304 atomic_wr(wr)->compare_add);
3305 } else {
3306 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
3307 DMA_REGPAIR_LE(awqe3->swap_data,
3308 atomic_wr(wr)->swap);
3309 DMA_REGPAIR_LE(awqe3->cmp_data,
3310 atomic_wr(wr)->compare_add);
3311 }
3312
3313 qedr_prepare_sq_sges(qp, NULL, wr);
3314
3315 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
3316 qp->prev_wqe_size = awqe1->wqe_size;
3317 break;
3318
3319 case IB_WR_LOCAL_INV:
3320 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
3321 iwqe->wqe_size = 1;
3322
3323 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
3324 iwqe->inv_l_key = wr->ex.invalidate_rkey;
3325 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
3326 qp->prev_wqe_size = iwqe->wqe_size;
3327 break;
3328 case IB_WR_REG_MR:
3329 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
3330 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
3331 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
3332 fwqe1->wqe_size = 2;
3333
3334 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
3335 if (rc) {
3336 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
3337 *bad_wr = wr;
3338 break;
3339 }
3340
3341 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
3342 qp->prev_wqe_size = fwqe1->wqe_size;
3343 break;
3344 default:
3345 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
3346 rc = -EINVAL;
3347 *bad_wr = wr;
3348 break;
3349 }
3350
3351 if (*bad_wr) {
3352 u16 value;
3353
3354
3355
3356
3357 value = le16_to_cpu(qp->sq.db_data.data.value);
3358 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
3359
3360
3361 qp->prev_wqe_size = wqe->prev_wqe_size;
3362 rc = -EINVAL;
3363 DP_ERR(dev, "POST SEND FAILED\n");
3364 }
3365
3366 return rc;
3367}
3368
3369int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3370 const struct ib_send_wr **bad_wr)
3371{
3372 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3373 struct qedr_qp *qp = get_qedr_qp(ibqp);
3374 unsigned long flags;
3375 int rc = 0;
3376
3377 *bad_wr = NULL;
3378
3379 if (qp->qp_type == IB_QPT_GSI)
3380 return qedr_gsi_post_send(ibqp, wr, bad_wr);
3381
3382 spin_lock_irqsave(&qp->q_lock, flags);
3383
3384 if (rdma_protocol_roce(&dev->ibdev, 1)) {
3385 if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3386 (qp->state != QED_ROCE_QP_STATE_ERR) &&
3387 (qp->state != QED_ROCE_QP_STATE_SQD)) {
3388 spin_unlock_irqrestore(&qp->q_lock, flags);
3389 *bad_wr = wr;
3390 DP_DEBUG(dev, QEDR_MSG_CQ,
3391 "QP in wrong state! QP icid=0x%x state %d\n",
3392 qp->icid, qp->state);
3393 return -EINVAL;
3394 }
3395 }
3396
3397 while (wr) {
3398 rc = __qedr_post_send(ibqp, wr, bad_wr);
3399 if (rc)
3400 break;
3401
3402 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3403
3404 qedr_inc_sw_prod(&qp->sq);
3405
3406 qp->sq.db_data.data.value++;
3407
3408 wr = wr->next;
3409 }
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424 smp_wmb();
3425 writel(qp->sq.db_data.raw, qp->sq.db);
3426
3427 spin_unlock_irqrestore(&qp->q_lock, flags);
3428
3429 return rc;
3430}
3431
3432static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq)
3433{
3434 u32 used;
3435
3436
3437
3438
3439
3440 used = hw_srq->wr_prod_cnt - hw_srq->wr_cons_cnt;
3441
3442 return hw_srq->max_wr - used;
3443}
3444
3445int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
3446 const struct ib_recv_wr **bad_wr)
3447{
3448 struct qedr_srq *srq = get_qedr_srq(ibsrq);
3449 struct qedr_srq_hwq_info *hw_srq;
3450 struct qedr_dev *dev = srq->dev;
3451 struct qed_chain *pbl;
3452 unsigned long flags;
3453 int status = 0;
3454 u32 num_sge;
3455 u32 offset;
3456
3457 spin_lock_irqsave(&srq->lock, flags);
3458
3459 hw_srq = &srq->hw_srq;
3460 pbl = &srq->hw_srq.pbl;
3461 while (wr) {
3462 struct rdma_srq_wqe_header *hdr;
3463 int i;
3464
3465 if (!qedr_srq_elem_left(hw_srq) ||
3466 wr->num_sge > srq->hw_srq.max_sges) {
3467 DP_ERR(dev, "Can't post WR (%d,%d) || (%d > %d)\n",
3468 hw_srq->wr_prod_cnt, hw_srq->wr_cons_cnt,
3469 wr->num_sge, srq->hw_srq.max_sges);
3470 status = -ENOMEM;
3471 *bad_wr = wr;
3472 break;
3473 }
3474
3475 hdr = qed_chain_produce(pbl);
3476 num_sge = wr->num_sge;
3477
3478 SRQ_HDR_SET(hdr, wr->wr_id, num_sge);
3479
3480 srq->hw_srq.wr_prod_cnt++;
3481 hw_srq->wqe_prod++;
3482 hw_srq->sge_prod++;
3483
3484 DP_DEBUG(dev, QEDR_MSG_SRQ,
3485 "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n",
3486 wr->num_sge, hw_srq->wqe_prod, wr->wr_id);
3487
3488 for (i = 0; i < wr->num_sge; i++) {
3489 struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl);
3490
3491
3492 SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,
3493 wr->sg_list[i].length, wr->sg_list[i].lkey);
3494
3495 DP_DEBUG(dev, QEDR_MSG_SRQ,
3496 "[%d]: len %d key %x addr %x:%x\n",
3497 i, srq_sge->length, srq_sge->l_key,
3498 srq_sge->addr.hi, srq_sge->addr.lo);
3499 hw_srq->sge_prod++;
3500 }
3501
3502
3503
3504
3505 wmb();
3506
3507
3508
3509
3510
3511 *srq->hw_srq.virt_prod_pair_addr = hw_srq->sge_prod;
3512 offset = offsetof(struct rdma_srq_producers, wqe_prod);
3513 *((u8 *)srq->hw_srq.virt_prod_pair_addr + offset) =
3514 hw_srq->wqe_prod;
3515
3516
3517 wmb();
3518 wr = wr->next;
3519 }
3520
3521 DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n",
3522 qed_chain_get_elem_left(pbl));
3523 spin_unlock_irqrestore(&srq->lock, flags);
3524
3525 return status;
3526}
3527
3528int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
3529 const struct ib_recv_wr **bad_wr)
3530{
3531 struct qedr_qp *qp = get_qedr_qp(ibqp);
3532 struct qedr_dev *dev = qp->dev;
3533 unsigned long flags;
3534 int status = 0;
3535
3536 if (qp->qp_type == IB_QPT_GSI)
3537 return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3538
3539 spin_lock_irqsave(&qp->q_lock, flags);
3540
3541 if (qp->state == QED_ROCE_QP_STATE_RESET) {
3542 spin_unlock_irqrestore(&qp->q_lock, flags);
3543 *bad_wr = wr;
3544 return -EINVAL;
3545 }
3546
3547 while (wr) {
3548 int i;
3549
3550 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3551 QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3552 wr->num_sge > qp->rq.max_sges) {
3553 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n",
3554 qed_chain_get_elem_left_u32(&qp->rq.pbl),
3555 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3556 qp->rq.max_sges);
3557 status = -ENOMEM;
3558 *bad_wr = wr;
3559 break;
3560 }
3561 for (i = 0; i < wr->num_sge; i++) {
3562 u32 flags = 0;
3563 struct rdma_rq_sge *rqe =
3564 qed_chain_produce(&qp->rq.pbl);
3565
3566
3567
3568
3569 if (!i)
3570 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3571 wr->num_sge);
3572
3573 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO,
3574 wr->sg_list[i].lkey);
3575
3576 RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3577 wr->sg_list[i].length, flags);
3578 }
3579
3580
3581
3582
3583
3584 if (!wr->num_sge) {
3585 u32 flags = 0;
3586 struct rdma_rq_sge *rqe =
3587 qed_chain_produce(&qp->rq.pbl);
3588
3589
3590
3591
3592 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0);
3593 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3594
3595 RQ_SGE_SET(rqe, 0, 0, flags);
3596 i = 1;
3597 }
3598
3599 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3600 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3601
3602 qedr_inc_sw_prod(&qp->rq);
3603
3604
3605
3606
3607
3608
3609
3610
3611 smp_wmb();
3612
3613 qp->rq.db_data.data.value++;
3614
3615 writel(qp->rq.db_data.raw, qp->rq.db);
3616
3617 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
3618 writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
3619 }
3620
3621 wr = wr->next;
3622 }
3623
3624 spin_unlock_irqrestore(&qp->q_lock, flags);
3625
3626 return status;
3627}
3628
3629static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
3630{
3631 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3632
3633 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
3634 cq->pbl_toggle;
3635}
3636
3637static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
3638{
3639 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3640 struct qedr_qp *qp;
3641
3642 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
3643 resp_cqe->qp_handle.lo,
3644 u64);
3645 return qp;
3646}
3647
3648static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
3649{
3650 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3651
3652 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
3653}
3654
3655
3656static union rdma_cqe *get_cqe(struct qedr_cq *cq)
3657{
3658 return cq->latest_cqe;
3659}
3660
3661
3662
3663
3664
3665
3666
3667
3668static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
3669{
3670 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
3671 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3672}
3673
3674static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
3675 struct qedr_cq *cq, int num_entries,
3676 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
3677 int force)
3678{
3679 u16 cnt = 0;
3680
3681 while (num_entries && qp->sq.wqe_cons != hw_cons) {
3682 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
3683 qedr_chk_if_fmr(qp);
3684
3685 goto next_cqe;
3686 }
3687
3688
3689 wc->status = status;
3690 wc->vendor_err = 0;
3691 wc->wc_flags = 0;
3692 wc->src_qp = qp->id;
3693 wc->qp = &qp->ibqp;
3694
3695 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
3696 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
3697
3698 switch (wc->opcode) {
3699 case IB_WC_RDMA_WRITE:
3700 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3701 break;
3702 case IB_WC_COMP_SWAP:
3703 case IB_WC_FETCH_ADD:
3704 wc->byte_len = 8;
3705 break;
3706 case IB_WC_REG_MR:
3707 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3708 break;
3709 case IB_WC_RDMA_READ:
3710 case IB_WC_SEND:
3711 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3712 break;
3713 default:
3714 break;
3715 }
3716
3717 num_entries--;
3718 wc++;
3719 cnt++;
3720next_cqe:
3721 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
3722 qed_chain_consume(&qp->sq.pbl);
3723 qedr_inc_sw_cons(&qp->sq);
3724 }
3725
3726 return cnt;
3727}
3728
3729static int qedr_poll_cq_req(struct qedr_dev *dev,
3730 struct qedr_qp *qp, struct qedr_cq *cq,
3731 int num_entries, struct ib_wc *wc,
3732 struct rdma_cqe_requester *req)
3733{
3734 int cnt = 0;
3735
3736 switch (req->status) {
3737 case RDMA_CQE_REQ_STS_OK:
3738 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3739 IB_WC_SUCCESS, 0);
3740 break;
3741 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
3742 if (qp->state != QED_ROCE_QP_STATE_ERR)
3743 DP_DEBUG(dev, QEDR_MSG_CQ,
3744 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3745 cq->icid, qp->icid);
3746 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3747 IB_WC_WR_FLUSH_ERR, 1);
3748 break;
3749 default:
3750
3751 qp->state = QED_ROCE_QP_STATE_ERR;
3752 cnt = process_req(dev, qp, cq, num_entries, wc,
3753 req->sq_cons - 1, IB_WC_SUCCESS, 0);
3754 wc += cnt;
3755
3756 if (cnt < num_entries) {
3757 enum ib_wc_status wc_status;
3758
3759 switch (req->status) {
3760 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
3761 DP_ERR(dev,
3762 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3763 cq->icid, qp->icid);
3764 wc_status = IB_WC_BAD_RESP_ERR;
3765 break;
3766 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
3767 DP_ERR(dev,
3768 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3769 cq->icid, qp->icid);
3770 wc_status = IB_WC_LOC_LEN_ERR;
3771 break;
3772 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
3773 DP_ERR(dev,
3774 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3775 cq->icid, qp->icid);
3776 wc_status = IB_WC_LOC_QP_OP_ERR;
3777 break;
3778 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
3779 DP_ERR(dev,
3780 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3781 cq->icid, qp->icid);
3782 wc_status = IB_WC_LOC_PROT_ERR;
3783 break;
3784 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
3785 DP_ERR(dev,
3786 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3787 cq->icid, qp->icid);
3788 wc_status = IB_WC_MW_BIND_ERR;
3789 break;
3790 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
3791 DP_ERR(dev,
3792 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3793 cq->icid, qp->icid);
3794 wc_status = IB_WC_REM_INV_REQ_ERR;
3795 break;
3796 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
3797 DP_ERR(dev,
3798 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3799 cq->icid, qp->icid);
3800 wc_status = IB_WC_REM_ACCESS_ERR;
3801 break;
3802 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
3803 DP_ERR(dev,
3804 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3805 cq->icid, qp->icid);
3806 wc_status = IB_WC_REM_OP_ERR;
3807 break;
3808 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
3809 DP_ERR(dev,
3810 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3811 cq->icid, qp->icid);
3812 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
3813 break;
3814 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
3815 DP_ERR(dev,
3816 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3817 cq->icid, qp->icid);
3818 wc_status = IB_WC_RETRY_EXC_ERR;
3819 break;
3820 default:
3821 DP_ERR(dev,
3822 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3823 cq->icid, qp->icid);
3824 wc_status = IB_WC_GENERAL_ERR;
3825 }
3826 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
3827 wc_status, 1);
3828 }
3829 }
3830
3831 return cnt;
3832}
3833
3834static inline int qedr_cqe_resp_status_to_ib(u8 status)
3835{
3836 switch (status) {
3837 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
3838 return IB_WC_LOC_ACCESS_ERR;
3839 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
3840 return IB_WC_LOC_LEN_ERR;
3841 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
3842 return IB_WC_LOC_QP_OP_ERR;
3843 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
3844 return IB_WC_LOC_PROT_ERR;
3845 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
3846 return IB_WC_MW_BIND_ERR;
3847 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
3848 return IB_WC_REM_INV_RD_REQ_ERR;
3849 case RDMA_CQE_RESP_STS_OK:
3850 return IB_WC_SUCCESS;
3851 default:
3852 return IB_WC_GENERAL_ERR;
3853 }
3854}
3855
3856static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp,
3857 struct ib_wc *wc)
3858{
3859 wc->status = IB_WC_SUCCESS;
3860 wc->byte_len = le32_to_cpu(resp->length);
3861
3862 if (resp->flags & QEDR_RESP_IMM) {
3863 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key));
3864 wc->wc_flags |= IB_WC_WITH_IMM;
3865
3866 if (resp->flags & QEDR_RESP_RDMA)
3867 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
3868
3869 if (resp->flags & QEDR_RESP_INV)
3870 return -EINVAL;
3871
3872 } else if (resp->flags & QEDR_RESP_INV) {
3873 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
3874 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
3875
3876 if (resp->flags & QEDR_RESP_RDMA)
3877 return -EINVAL;
3878
3879 } else if (resp->flags & QEDR_RESP_RDMA) {
3880 return -EINVAL;
3881 }
3882
3883 return 0;
3884}
3885
3886static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3887 struct qedr_cq *cq, struct ib_wc *wc,
3888 struct rdma_cqe_responder *resp, u64 wr_id)
3889{
3890
3891 wc->opcode = IB_WC_RECV;
3892 wc->wc_flags = 0;
3893
3894 if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) {
3895 if (qedr_set_ok_cqe_resp_wc(resp, wc))
3896 DP_ERR(dev,
3897 "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n",
3898 cq, cq->icid, resp->flags);
3899
3900 } else {
3901 wc->status = qedr_cqe_resp_status_to_ib(resp->status);
3902 if (wc->status == IB_WC_GENERAL_ERR)
3903 DP_ERR(dev,
3904 "CQ %p (icid=%d) contains an invalid CQE status %d\n",
3905 cq, cq->icid, resp->status);
3906 }
3907
3908
3909 wc->vendor_err = 0;
3910 wc->src_qp = qp->id;
3911 wc->qp = &qp->ibqp;
3912 wc->wr_id = wr_id;
3913}
3914
3915static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp,
3916 struct qedr_cq *cq, struct ib_wc *wc,
3917 struct rdma_cqe_responder *resp)
3918{
3919 struct qedr_srq *srq = qp->srq;
3920 u64 wr_id;
3921
3922 wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi),
3923 le32_to_cpu(resp->srq_wr_id.lo), u64);
3924
3925 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
3926 wc->status = IB_WC_WR_FLUSH_ERR;
3927 wc->vendor_err = 0;
3928 wc->wr_id = wr_id;
3929 wc->byte_len = 0;
3930 wc->src_qp = qp->id;
3931 wc->qp = &qp->ibqp;
3932 wc->wr_id = wr_id;
3933 } else {
3934 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
3935 }
3936 srq->hw_srq.wr_cons_cnt++;
3937
3938 return 1;
3939}
3940static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3941 struct qedr_cq *cq, struct ib_wc *wc,
3942 struct rdma_cqe_responder *resp)
3943{
3944 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
3945
3946 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
3947
3948 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
3949 qed_chain_consume(&qp->rq.pbl);
3950 qedr_inc_sw_cons(&qp->rq);
3951
3952 return 1;
3953}
3954
3955static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
3956 int num_entries, struct ib_wc *wc, u16 hw_cons)
3957{
3958 u16 cnt = 0;
3959
3960 while (num_entries && qp->rq.wqe_cons != hw_cons) {
3961
3962 wc->status = IB_WC_WR_FLUSH_ERR;
3963 wc->vendor_err = 0;
3964 wc->wc_flags = 0;
3965 wc->src_qp = qp->id;
3966 wc->byte_len = 0;
3967 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
3968 wc->qp = &qp->ibqp;
3969 num_entries--;
3970 wc++;
3971 cnt++;
3972 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
3973 qed_chain_consume(&qp->rq.pbl);
3974 qedr_inc_sw_cons(&qp->rq);
3975 }
3976
3977 return cnt;
3978}
3979
3980static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
3981 struct rdma_cqe_responder *resp, int *update)
3982{
3983 if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) {
3984 consume_cqe(cq);
3985 *update |= 1;
3986 }
3987}
3988
3989static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp,
3990 struct qedr_cq *cq, int num_entries,
3991 struct ib_wc *wc,
3992 struct rdma_cqe_responder *resp)
3993{
3994 int cnt;
3995
3996 cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
3997 consume_cqe(cq);
3998
3999 return cnt;
4000}
4001
4002static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
4003 struct qedr_cq *cq, int num_entries,
4004 struct ib_wc *wc, struct rdma_cqe_responder *resp,
4005 int *update)
4006{
4007 int cnt;
4008
4009 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4010 cnt = process_resp_flush(qp, cq, num_entries, wc,
4011 resp->rq_cons_or_srq_id);
4012 try_consume_resp_cqe(cq, qp, resp, update);
4013 } else {
4014 cnt = process_resp_one(dev, qp, cq, wc, resp);
4015 consume_cqe(cq);
4016 *update |= 1;
4017 }
4018
4019 return cnt;
4020}
4021
4022static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4023 struct rdma_cqe_requester *req, int *update)
4024{
4025 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
4026 consume_cqe(cq);
4027 *update |= 1;
4028 }
4029}
4030
4031int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
4032{
4033 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
4034 struct qedr_cq *cq = get_qedr_cq(ibcq);
4035 union rdma_cqe *cqe;
4036 u32 old_cons, new_cons;
4037 unsigned long flags;
4038 int update = 0;
4039 int done = 0;
4040
4041 if (cq->destroyed) {
4042 DP_ERR(dev,
4043 "warning: poll was invoked after destroy for cq %p (icid=%d)\n",
4044 cq, cq->icid);
4045 return 0;
4046 }
4047
4048 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
4049 return qedr_gsi_poll_cq(ibcq, num_entries, wc);
4050
4051 spin_lock_irqsave(&cq->cq_lock, flags);
4052 cqe = cq->latest_cqe;
4053 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4054 while (num_entries && is_valid_cqe(cq, cqe)) {
4055 struct qedr_qp *qp;
4056 int cnt = 0;
4057
4058
4059 rmb();
4060
4061 qp = cqe_get_qp(cqe);
4062 if (!qp) {
4063 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
4064 break;
4065 }
4066
4067 wc->qp = &qp->ibqp;
4068
4069 switch (cqe_get_type(cqe)) {
4070 case RDMA_CQE_TYPE_REQUESTER:
4071 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
4072 &cqe->req);
4073 try_consume_req_cqe(cq, qp, &cqe->req, &update);
4074 break;
4075 case RDMA_CQE_TYPE_RESPONDER_RQ:
4076 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
4077 &cqe->resp, &update);
4078 break;
4079 case RDMA_CQE_TYPE_RESPONDER_SRQ:
4080 cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries,
4081 wc, &cqe->resp);
4082 update = 1;
4083 break;
4084 case RDMA_CQE_TYPE_INVALID:
4085 default:
4086 DP_ERR(dev, "Error: invalid CQE type = %d\n",
4087 cqe_get_type(cqe));
4088 }
4089 num_entries -= cnt;
4090 wc += cnt;
4091 done += cnt;
4092
4093 cqe = get_cqe(cq);
4094 }
4095 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4096
4097 cq->cq_cons += new_cons - old_cons;
4098
4099 if (update)
4100
4101
4102
4103 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
4104
4105 spin_unlock_irqrestore(&cq->cq_lock, flags);
4106 return done;
4107}
4108
4109int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
4110 u8 port_num,
4111 const struct ib_wc *in_wc,
4112 const struct ib_grh *in_grh,
4113 const struct ib_mad_hdr *mad_hdr,
4114 size_t in_mad_size, struct ib_mad_hdr *out_mad,
4115 size_t *out_mad_size, u16 *out_mad_pkey_index)
4116{
4117 struct qedr_dev *dev = get_qedr_dev(ibdev);
4118
4119 DP_DEBUG(dev, QEDR_MSG_GSI,
4120 "QEDR_PROCESS_MAD in_mad %x %x %x %x %x %x %x %x\n",
4121 mad_hdr->attr_id, mad_hdr->base_version, mad_hdr->attr_mod,
4122 mad_hdr->class_specific, mad_hdr->class_version,
4123 mad_hdr->method, mad_hdr->mgmt_class, mad_hdr->status);
4124 return IB_MAD_RESULT_SUCCESS;
4125}
4126