1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32#include <linux/module.h>
33#include <linux/moduleparam.h>
34#include <linux/device.h>
35#include <linux/netdevice.h>
36#include <linux/etherdevice.h>
37#include <linux/delay.h>
38#include <linux/errno.h>
39#include <linux/list.h>
40#include <linux/sched/mm.h>
41#include <linux/spinlock.h>
42#include <linux/ethtool.h>
43#include <linux/rtnetlink.h>
44#include <linux/inetdevice.h>
45#include <linux/slab.h>
46
47#include <asm/io.h>
48#include <asm/irq.h>
49#include <asm/byteorder.h>
50
51#include <rdma/iw_cm.h>
52#include <rdma/ib_verbs.h>
53#include <rdma/ib_smi.h>
54#include <rdma/ib_umem.h>
55#include <rdma/ib_user_verbs.h>
56
57#include "cxio_hal.h"
58#include "iwch.h"
59#include "iwch_provider.h"
60#include "iwch_cm.h"
61#include <rdma/cxgb3-abi.h>
62#include "common.h"
63
64static struct ib_ah *iwch_ah_create(struct ib_pd *pd,
65 struct rdma_ah_attr *ah_attr,
66 struct ib_udata *udata)
67{
68 return ERR_PTR(-ENOSYS);
69}
70
71static int iwch_ah_destroy(struct ib_ah *ah)
72{
73 return -ENOSYS;
74}
75
76static int iwch_multicast_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
77{
78 return -ENOSYS;
79}
80
81static int iwch_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
82{
83 return -ENOSYS;
84}
85
86static int iwch_process_mad(struct ib_device *ibdev,
87 int mad_flags,
88 u8 port_num,
89 const struct ib_wc *in_wc,
90 const struct ib_grh *in_grh,
91 const struct ib_mad_hdr *in_mad,
92 size_t in_mad_size,
93 struct ib_mad_hdr *out_mad,
94 size_t *out_mad_size,
95 u16 *out_mad_pkey_index)
96{
97 return -ENOSYS;
98}
99
100static int iwch_dealloc_ucontext(struct ib_ucontext *context)
101{
102 struct iwch_dev *rhp = to_iwch_dev(context->device);
103 struct iwch_ucontext *ucontext = to_iwch_ucontext(context);
104 struct iwch_mm_entry *mm, *tmp;
105
106 pr_debug("%s context %p\n", __func__, context);
107 list_for_each_entry_safe(mm, tmp, &ucontext->mmaps, entry)
108 kfree(mm);
109 cxio_release_ucontext(&rhp->rdev, &ucontext->uctx);
110 kfree(ucontext);
111 return 0;
112}
113
114static struct ib_ucontext *iwch_alloc_ucontext(struct ib_device *ibdev,
115 struct ib_udata *udata)
116{
117 struct iwch_ucontext *context;
118 struct iwch_dev *rhp = to_iwch_dev(ibdev);
119
120 pr_debug("%s ibdev %p\n", __func__, ibdev);
121 context = kzalloc(sizeof(*context), GFP_KERNEL);
122 if (!context)
123 return ERR_PTR(-ENOMEM);
124 cxio_init_ucontext(&rhp->rdev, &context->uctx);
125 INIT_LIST_HEAD(&context->mmaps);
126 spin_lock_init(&context->mmap_lock);
127 return &context->ibucontext;
128}
129
130static int iwch_destroy_cq(struct ib_cq *ib_cq)
131{
132 struct iwch_cq *chp;
133
134 pr_debug("%s ib_cq %p\n", __func__, ib_cq);
135 chp = to_iwch_cq(ib_cq);
136
137 remove_handle(chp->rhp, &chp->rhp->cqidr, chp->cq.cqid);
138 atomic_dec(&chp->refcnt);
139 wait_event(chp->wait, !atomic_read(&chp->refcnt));
140
141 cxio_destroy_cq(&chp->rhp->rdev, &chp->cq);
142 kfree(chp);
143 return 0;
144}
145
146static struct ib_cq *iwch_create_cq(struct ib_device *ibdev,
147 const struct ib_cq_init_attr *attr,
148 struct ib_ucontext *ib_context,
149 struct ib_udata *udata)
150{
151 int entries = attr->cqe;
152 struct iwch_dev *rhp;
153 struct iwch_cq *chp;
154 struct iwch_create_cq_resp uresp;
155 struct iwch_create_cq_req ureq;
156 struct iwch_ucontext *ucontext = NULL;
157 static int warned;
158 size_t resplen;
159
160 pr_debug("%s ib_dev %p entries %d\n", __func__, ibdev, entries);
161 if (attr->flags)
162 return ERR_PTR(-EINVAL);
163
164 rhp = to_iwch_dev(ibdev);
165 chp = kzalloc(sizeof(*chp), GFP_KERNEL);
166 if (!chp)
167 return ERR_PTR(-ENOMEM);
168
169 if (ib_context) {
170 ucontext = to_iwch_ucontext(ib_context);
171 if (!t3a_device(rhp)) {
172 if (ib_copy_from_udata(&ureq, udata, sizeof (ureq))) {
173 kfree(chp);
174 return ERR_PTR(-EFAULT);
175 }
176 chp->user_rptr_addr = (u32 __user *)(unsigned long)ureq.user_rptr_addr;
177 }
178 }
179
180 if (t3a_device(rhp)) {
181
182
183
184
185
186
187
188
189
190
191 entries += 16;
192 }
193 entries = roundup_pow_of_two(entries);
194 chp->cq.size_log2 = ilog2(entries);
195
196 if (cxio_create_cq(&rhp->rdev, &chp->cq, !ucontext)) {
197 kfree(chp);
198 return ERR_PTR(-ENOMEM);
199 }
200 chp->rhp = rhp;
201 chp->ibcq.cqe = 1 << chp->cq.size_log2;
202 spin_lock_init(&chp->lock);
203 spin_lock_init(&chp->comp_handler_lock);
204 atomic_set(&chp->refcnt, 1);
205 init_waitqueue_head(&chp->wait);
206 if (insert_handle(rhp, &rhp->cqidr, chp, chp->cq.cqid)) {
207 cxio_destroy_cq(&chp->rhp->rdev, &chp->cq);
208 kfree(chp);
209 return ERR_PTR(-ENOMEM);
210 }
211
212 if (ucontext) {
213 struct iwch_mm_entry *mm;
214
215 mm = kmalloc(sizeof *mm, GFP_KERNEL);
216 if (!mm) {
217 iwch_destroy_cq(&chp->ibcq);
218 return ERR_PTR(-ENOMEM);
219 }
220 uresp.cqid = chp->cq.cqid;
221 uresp.size_log2 = chp->cq.size_log2;
222 spin_lock(&ucontext->mmap_lock);
223 uresp.key = ucontext->key;
224 ucontext->key += PAGE_SIZE;
225 spin_unlock(&ucontext->mmap_lock);
226 mm->key = uresp.key;
227 mm->addr = virt_to_phys(chp->cq.queue);
228 if (udata->outlen < sizeof uresp) {
229 if (!warned++)
230 pr_warn("Warning - downlevel libcxgb3 (non-fatal)\n");
231 mm->len = PAGE_ALIGN((1UL << uresp.size_log2) *
232 sizeof(struct t3_cqe));
233 resplen = sizeof(struct iwch_create_cq_resp_v0);
234 } else {
235 mm->len = PAGE_ALIGN(((1UL << uresp.size_log2) + 1) *
236 sizeof(struct t3_cqe));
237 uresp.memsize = mm->len;
238 uresp.reserved = 0;
239 resplen = sizeof uresp;
240 }
241 if (ib_copy_to_udata(udata, &uresp, resplen)) {
242 kfree(mm);
243 iwch_destroy_cq(&chp->ibcq);
244 return ERR_PTR(-EFAULT);
245 }
246 insert_mmap(ucontext, mm);
247 }
248 pr_debug("created cqid 0x%0x chp %p size 0x%0x, dma_addr 0x%0llx\n",
249 chp->cq.cqid, chp, (1 << chp->cq.size_log2),
250 (unsigned long long)chp->cq.dma_addr);
251 return &chp->ibcq;
252}
253
254static int iwch_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata)
255{
256#ifdef notyet
257 struct iwch_cq *chp = to_iwch_cq(cq);
258 struct t3_cq oldcq, newcq;
259 int ret;
260
261 pr_debug("%s ib_cq %p cqe %d\n", __func__, cq, cqe);
262
263
264 if (cqe <= cq->cqe)
265 return 0;
266
267
268 cqe = roundup_pow_of_two(cqe+1);
269 newcq.size_log2 = ilog2(cqe);
270
271
272 if (cqe < Q_COUNT(chp->cq.rptr, chp->cq.wptr)) {
273 return -ENOMEM;
274 }
275
276
277 ret = iwch_quiesce_qps(chp);
278 if (ret) {
279 return ret;
280 }
281
282 ret = cxio_create_cq(&chp->rhp->rdev, &newcq);
283 if (ret) {
284 return ret;
285 }
286
287
288 memcpy(newcq.queue, chp->cq.queue, (1 << chp->cq.size_log2) *
289 sizeof(struct t3_cqe));
290
291
292 oldcq = chp->cq;
293 chp->cq = newcq;
294 chp->cq.cqid = oldcq.cqid;
295
296
297 ret = cxio_resize_cq(&chp->rhp->rdev, &chp->cq);
298 if (ret) {
299 chp->cq = oldcq;
300 return ret;
301 }
302 chp->ibcq.cqe = (1<<chp->cq.size_log2) - 1;
303
304
305 oldcq.cqid = newcq.cqid;
306 ret = cxio_destroy_cq(&chp->rhp->rdev, &oldcq);
307 if (ret) {
308 pr_err("%s - cxio_destroy_cq failed %d\n", __func__, ret);
309 }
310
311
312
313
314 ret = iwch_resume_qps(chp);
315 return ret;
316#else
317 return -ENOSYS;
318#endif
319}
320
321static int iwch_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
322{
323 struct iwch_dev *rhp;
324 struct iwch_cq *chp;
325 enum t3_cq_opcode cq_op;
326 int err;
327 unsigned long flag;
328 u32 rptr;
329
330 chp = to_iwch_cq(ibcq);
331 rhp = chp->rhp;
332 if ((flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED)
333 cq_op = CQ_ARM_SE;
334 else
335 cq_op = CQ_ARM_AN;
336 if (chp->user_rptr_addr) {
337 if (get_user(rptr, chp->user_rptr_addr))
338 return -EFAULT;
339 spin_lock_irqsave(&chp->lock, flag);
340 chp->cq.rptr = rptr;
341 } else
342 spin_lock_irqsave(&chp->lock, flag);
343 pr_debug("%s rptr 0x%x\n", __func__, chp->cq.rptr);
344 err = cxio_hal_cq_op(&rhp->rdev, &chp->cq, cq_op, 0);
345 spin_unlock_irqrestore(&chp->lock, flag);
346 if (err < 0)
347 pr_err("Error %d rearming CQID 0x%x\n", err, chp->cq.cqid);
348 if (err > 0 && !(flags & IB_CQ_REPORT_MISSED_EVENTS))
349 err = 0;
350 return err;
351}
352
353static int iwch_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
354{
355 int len = vma->vm_end - vma->vm_start;
356 u32 key = vma->vm_pgoff << PAGE_SHIFT;
357 struct cxio_rdev *rdev_p;
358 int ret = 0;
359 struct iwch_mm_entry *mm;
360 struct iwch_ucontext *ucontext;
361 u64 addr;
362
363 pr_debug("%s pgoff 0x%lx key 0x%x len %d\n", __func__, vma->vm_pgoff,
364 key, len);
365
366 if (vma->vm_start & (PAGE_SIZE-1)) {
367 return -EINVAL;
368 }
369
370 rdev_p = &(to_iwch_dev(context->device)->rdev);
371 ucontext = to_iwch_ucontext(context);
372
373 mm = remove_mmap(ucontext, key, len);
374 if (!mm)
375 return -EINVAL;
376 addr = mm->addr;
377 kfree(mm);
378
379 if ((addr >= rdev_p->rnic_info.udbell_physbase) &&
380 (addr < (rdev_p->rnic_info.udbell_physbase +
381 rdev_p->rnic_info.udbell_len))) {
382
383
384
385
386 if (vma->vm_flags & VM_READ) {
387 return -EPERM;
388 }
389
390 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
391 vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND;
392 vma->vm_flags &= ~VM_MAYREAD;
393 ret = io_remap_pfn_range(vma, vma->vm_start,
394 addr >> PAGE_SHIFT,
395 len, vma->vm_page_prot);
396 } else {
397
398
399
400
401 ret = remap_pfn_range(vma, vma->vm_start,
402 addr >> PAGE_SHIFT,
403 len, vma->vm_page_prot);
404 }
405
406 return ret;
407}
408
409static int iwch_deallocate_pd(struct ib_pd *pd)
410{
411 struct iwch_dev *rhp;
412 struct iwch_pd *php;
413
414 php = to_iwch_pd(pd);
415 rhp = php->rhp;
416 pr_debug("%s ibpd %p pdid 0x%x\n", __func__, pd, php->pdid);
417 cxio_hal_put_pdid(rhp->rdev.rscp, php->pdid);
418 kfree(php);
419 return 0;
420}
421
422static struct ib_pd *iwch_allocate_pd(struct ib_device *ibdev,
423 struct ib_ucontext *context,
424 struct ib_udata *udata)
425{
426 struct iwch_pd *php;
427 u32 pdid;
428 struct iwch_dev *rhp;
429
430 pr_debug("%s ibdev %p\n", __func__, ibdev);
431 rhp = (struct iwch_dev *) ibdev;
432 pdid = cxio_hal_get_pdid(rhp->rdev.rscp);
433 if (!pdid)
434 return ERR_PTR(-EINVAL);
435 php = kzalloc(sizeof(*php), GFP_KERNEL);
436 if (!php) {
437 cxio_hal_put_pdid(rhp->rdev.rscp, pdid);
438 return ERR_PTR(-ENOMEM);
439 }
440 php->pdid = pdid;
441 php->rhp = rhp;
442 if (context) {
443 struct iwch_alloc_pd_resp resp = {.pdid = php->pdid};
444
445 if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
446 iwch_deallocate_pd(&php->ibpd);
447 return ERR_PTR(-EFAULT);
448 }
449 }
450 pr_debug("%s pdid 0x%0x ptr 0x%p\n", __func__, pdid, php);
451 return &php->ibpd;
452}
453
454static int iwch_dereg_mr(struct ib_mr *ib_mr)
455{
456 struct iwch_dev *rhp;
457 struct iwch_mr *mhp;
458 u32 mmid;
459
460 pr_debug("%s ib_mr %p\n", __func__, ib_mr);
461
462 mhp = to_iwch_mr(ib_mr);
463 kfree(mhp->pages);
464 rhp = mhp->rhp;
465 mmid = mhp->attr.stag >> 8;
466 cxio_dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size,
467 mhp->attr.pbl_addr);
468 iwch_free_pbl(mhp);
469 remove_handle(rhp, &rhp->mmidr, mmid);
470 if (mhp->kva)
471 kfree((void *) (unsigned long) mhp->kva);
472 if (mhp->umem)
473 ib_umem_release(mhp->umem);
474 pr_debug("%s mmid 0x%x ptr %p\n", __func__, mmid, mhp);
475 kfree(mhp);
476 return 0;
477}
478
479static struct ib_mr *iwch_get_dma_mr(struct ib_pd *pd, int acc)
480{
481 const u64 total_size = 0xffffffff;
482 const u64 mask = (total_size + PAGE_SIZE - 1) & PAGE_MASK;
483 struct iwch_pd *php = to_iwch_pd(pd);
484 struct iwch_dev *rhp = php->rhp;
485 struct iwch_mr *mhp;
486 __be64 *page_list;
487 int shift = 26, npages, ret, i;
488
489 pr_debug("%s ib_pd %p\n", __func__, pd);
490
491
492
493
494 if (sizeof(phys_addr_t) > 4) {
495 pr_warn_once("Cannot support dma_mrs on this platform\n");
496 return ERR_PTR(-ENOTSUPP);
497 }
498
499 mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
500 if (!mhp)
501 return ERR_PTR(-ENOMEM);
502
503 mhp->rhp = rhp;
504
505 npages = (total_size + (1ULL << shift) - 1) >> shift;
506 if (!npages) {
507 ret = -EINVAL;
508 goto err;
509 }
510
511 page_list = kmalloc_array(npages, sizeof(u64), GFP_KERNEL);
512 if (!page_list) {
513 ret = -ENOMEM;
514 goto err;
515 }
516
517 for (i = 0; i < npages; i++)
518 page_list[i] = cpu_to_be64((u64)i << shift);
519
520 pr_debug("%s mask 0x%llx shift %d len %lld pbl_size %d\n",
521 __func__, mask, shift, total_size, npages);
522
523 ret = iwch_alloc_pbl(mhp, npages);
524 if (ret) {
525 kfree(page_list);
526 goto err_pbl;
527 }
528
529 ret = iwch_write_pbl(mhp, page_list, npages, 0);
530 kfree(page_list);
531 if (ret)
532 goto err_pbl;
533
534 mhp->attr.pdid = php->pdid;
535 mhp->attr.zbva = 0;
536
537 mhp->attr.perms = iwch_ib_to_tpt_access(acc);
538 mhp->attr.va_fbo = 0;
539 mhp->attr.page_size = shift - 12;
540
541 mhp->attr.len = (u32) total_size;
542 mhp->attr.pbl_size = npages;
543 ret = iwch_register_mem(rhp, php, mhp, shift);
544 if (ret)
545 goto err_pbl;
546
547 return &mhp->ibmr;
548
549err_pbl:
550 iwch_free_pbl(mhp);
551
552err:
553 kfree(mhp);
554 return ERR_PTR(ret);
555}
556
557static struct ib_mr *iwch_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
558 u64 virt, int acc, struct ib_udata *udata)
559{
560 __be64 *pages;
561 int shift, n, len;
562 int i, k, entry;
563 int err = 0;
564 struct iwch_dev *rhp;
565 struct iwch_pd *php;
566 struct iwch_mr *mhp;
567 struct iwch_reg_user_mr_resp uresp;
568 struct scatterlist *sg;
569 pr_debug("%s ib_pd %p\n", __func__, pd);
570
571 php = to_iwch_pd(pd);
572 rhp = php->rhp;
573 mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
574 if (!mhp)
575 return ERR_PTR(-ENOMEM);
576
577 mhp->rhp = rhp;
578
579 mhp->umem = ib_umem_get(pd->uobject->context, start, length, acc, 0);
580 if (IS_ERR(mhp->umem)) {
581 err = PTR_ERR(mhp->umem);
582 kfree(mhp);
583 return ERR_PTR(err);
584 }
585
586 shift = mhp->umem->page_shift;
587
588 n = mhp->umem->nmap;
589
590 err = iwch_alloc_pbl(mhp, n);
591 if (err)
592 goto err;
593
594 pages = (__be64 *) __get_free_page(GFP_KERNEL);
595 if (!pages) {
596 err = -ENOMEM;
597 goto err_pbl;
598 }
599
600 i = n = 0;
601
602 for_each_sg(mhp->umem->sg_head.sgl, sg, mhp->umem->nmap, entry) {
603 len = sg_dma_len(sg) >> shift;
604 for (k = 0; k < len; ++k) {
605 pages[i++] = cpu_to_be64(sg_dma_address(sg) +
606 (k << shift));
607 if (i == PAGE_SIZE / sizeof *pages) {
608 err = iwch_write_pbl(mhp, pages, i, n);
609 if (err)
610 goto pbl_done;
611 n += i;
612 i = 0;
613 }
614 }
615 }
616
617 if (i)
618 err = iwch_write_pbl(mhp, pages, i, n);
619
620pbl_done:
621 free_page((unsigned long) pages);
622 if (err)
623 goto err_pbl;
624
625 mhp->attr.pdid = php->pdid;
626 mhp->attr.zbva = 0;
627 mhp->attr.perms = iwch_ib_to_tpt_access(acc);
628 mhp->attr.va_fbo = virt;
629 mhp->attr.page_size = shift - 12;
630 mhp->attr.len = (u32) length;
631
632 err = iwch_register_mem(rhp, php, mhp, shift);
633 if (err)
634 goto err_pbl;
635
636 if (udata && !t3a_device(rhp)) {
637 uresp.pbl_addr = (mhp->attr.pbl_addr -
638 rhp->rdev.rnic_info.pbl_base) >> 3;
639 pr_debug("%s user resp pbl_addr 0x%x\n", __func__,
640 uresp.pbl_addr);
641
642 if (ib_copy_to_udata(udata, &uresp, sizeof (uresp))) {
643 iwch_dereg_mr(&mhp->ibmr);
644 err = -EFAULT;
645 goto err;
646 }
647 }
648
649 return &mhp->ibmr;
650
651err_pbl:
652 iwch_free_pbl(mhp);
653
654err:
655 ib_umem_release(mhp->umem);
656 kfree(mhp);
657 return ERR_PTR(err);
658}
659
660static struct ib_mw *iwch_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
661 struct ib_udata *udata)
662{
663 struct iwch_dev *rhp;
664 struct iwch_pd *php;
665 struct iwch_mw *mhp;
666 u32 mmid;
667 u32 stag = 0;
668 int ret;
669
670 if (type != IB_MW_TYPE_1)
671 return ERR_PTR(-EINVAL);
672
673 php = to_iwch_pd(pd);
674 rhp = php->rhp;
675 mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
676 if (!mhp)
677 return ERR_PTR(-ENOMEM);
678 ret = cxio_allocate_window(&rhp->rdev, &stag, php->pdid);
679 if (ret) {
680 kfree(mhp);
681 return ERR_PTR(ret);
682 }
683 mhp->rhp = rhp;
684 mhp->attr.pdid = php->pdid;
685 mhp->attr.type = TPT_MW;
686 mhp->attr.stag = stag;
687 mmid = (stag) >> 8;
688 mhp->ibmw.rkey = stag;
689 if (insert_handle(rhp, &rhp->mmidr, mhp, mmid)) {
690 cxio_deallocate_window(&rhp->rdev, mhp->attr.stag);
691 kfree(mhp);
692 return ERR_PTR(-ENOMEM);
693 }
694 pr_debug("%s mmid 0x%x mhp %p stag 0x%x\n", __func__, mmid, mhp, stag);
695 return &(mhp->ibmw);
696}
697
698static int iwch_dealloc_mw(struct ib_mw *mw)
699{
700 struct iwch_dev *rhp;
701 struct iwch_mw *mhp;
702 u32 mmid;
703
704 mhp = to_iwch_mw(mw);
705 rhp = mhp->rhp;
706 mmid = (mw->rkey) >> 8;
707 cxio_deallocate_window(&rhp->rdev, mhp->attr.stag);
708 remove_handle(rhp, &rhp->mmidr, mmid);
709 pr_debug("%s ib_mw %p mmid 0x%x ptr %p\n", __func__, mw, mmid, mhp);
710 kfree(mhp);
711 return 0;
712}
713
714static struct ib_mr *iwch_alloc_mr(struct ib_pd *pd,
715 enum ib_mr_type mr_type,
716 u32 max_num_sg)
717{
718 struct iwch_dev *rhp;
719 struct iwch_pd *php;
720 struct iwch_mr *mhp;
721 u32 mmid;
722 u32 stag = 0;
723 int ret = -ENOMEM;
724
725 if (mr_type != IB_MR_TYPE_MEM_REG ||
726 max_num_sg > T3_MAX_FASTREG_DEPTH)
727 return ERR_PTR(-EINVAL);
728
729 php = to_iwch_pd(pd);
730 rhp = php->rhp;
731 mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
732 if (!mhp)
733 goto err;
734
735 mhp->pages = kcalloc(max_num_sg, sizeof(u64), GFP_KERNEL);
736 if (!mhp->pages)
737 goto pl_err;
738
739 mhp->rhp = rhp;
740 ret = iwch_alloc_pbl(mhp, max_num_sg);
741 if (ret)
742 goto err1;
743 mhp->attr.pbl_size = max_num_sg;
744 ret = cxio_allocate_stag(&rhp->rdev, &stag, php->pdid,
745 mhp->attr.pbl_size, mhp->attr.pbl_addr);
746 if (ret)
747 goto err2;
748 mhp->attr.pdid = php->pdid;
749 mhp->attr.type = TPT_NON_SHARED_MR;
750 mhp->attr.stag = stag;
751 mhp->attr.state = 1;
752 mmid = (stag) >> 8;
753 mhp->ibmr.rkey = mhp->ibmr.lkey = stag;
754 ret = insert_handle(rhp, &rhp->mmidr, mhp, mmid);
755 if (ret)
756 goto err3;
757
758 pr_debug("%s mmid 0x%x mhp %p stag 0x%x\n", __func__, mmid, mhp, stag);
759 return &(mhp->ibmr);
760err3:
761 cxio_dereg_mem(&rhp->rdev, stag, mhp->attr.pbl_size,
762 mhp->attr.pbl_addr);
763err2:
764 iwch_free_pbl(mhp);
765err1:
766 kfree(mhp->pages);
767pl_err:
768 kfree(mhp);
769err:
770 return ERR_PTR(ret);
771}
772
773static int iwch_set_page(struct ib_mr *ibmr, u64 addr)
774{
775 struct iwch_mr *mhp = to_iwch_mr(ibmr);
776
777 if (unlikely(mhp->npages == mhp->attr.pbl_size))
778 return -ENOMEM;
779
780 mhp->pages[mhp->npages++] = addr;
781
782 return 0;
783}
784
785static int iwch_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
786 int sg_nents, unsigned int *sg_offset)
787{
788 struct iwch_mr *mhp = to_iwch_mr(ibmr);
789
790 mhp->npages = 0;
791
792 return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, iwch_set_page);
793}
794
795static int iwch_destroy_qp(struct ib_qp *ib_qp)
796{
797 struct iwch_dev *rhp;
798 struct iwch_qp *qhp;
799 struct iwch_qp_attributes attrs;
800 struct iwch_ucontext *ucontext;
801
802 qhp = to_iwch_qp(ib_qp);
803 rhp = qhp->rhp;
804
805 attrs.next_state = IWCH_QP_STATE_ERROR;
806 iwch_modify_qp(rhp, qhp, IWCH_QP_ATTR_NEXT_STATE, &attrs, 0);
807 wait_event(qhp->wait, !qhp->ep);
808
809 remove_handle(rhp, &rhp->qpidr, qhp->wq.qpid);
810
811 atomic_dec(&qhp->refcnt);
812 wait_event(qhp->wait, !atomic_read(&qhp->refcnt));
813
814 ucontext = ib_qp->uobject ? to_iwch_ucontext(ib_qp->uobject->context)
815 : NULL;
816 cxio_destroy_qp(&rhp->rdev, &qhp->wq,
817 ucontext ? &ucontext->uctx : &rhp->rdev.uctx);
818
819 pr_debug("%s ib_qp %p qpid 0x%0x qhp %p\n", __func__,
820 ib_qp, qhp->wq.qpid, qhp);
821 kfree(qhp);
822 return 0;
823}
824
825static struct ib_qp *iwch_create_qp(struct ib_pd *pd,
826 struct ib_qp_init_attr *attrs,
827 struct ib_udata *udata)
828{
829 struct iwch_dev *rhp;
830 struct iwch_qp *qhp;
831 struct iwch_pd *php;
832 struct iwch_cq *schp;
833 struct iwch_cq *rchp;
834 struct iwch_create_qp_resp uresp;
835 int wqsize, sqsize, rqsize;
836 struct iwch_ucontext *ucontext;
837
838 pr_debug("%s ib_pd %p\n", __func__, pd);
839 if (attrs->qp_type != IB_QPT_RC)
840 return ERR_PTR(-EINVAL);
841 php = to_iwch_pd(pd);
842 rhp = php->rhp;
843 schp = get_chp(rhp, ((struct iwch_cq *) attrs->send_cq)->cq.cqid);
844 rchp = get_chp(rhp, ((struct iwch_cq *) attrs->recv_cq)->cq.cqid);
845 if (!schp || !rchp)
846 return ERR_PTR(-EINVAL);
847
848
849 rqsize = roundup_pow_of_two(attrs->cap.max_recv_wr);
850 if (rqsize == attrs->cap.max_recv_wr)
851 rqsize = roundup_pow_of_two(attrs->cap.max_recv_wr+1);
852
853
854 if (rqsize < 16)
855 rqsize = 16;
856
857 if (rqsize > T3_MAX_RQ_SIZE)
858 return ERR_PTR(-EINVAL);
859
860 if (attrs->cap.max_inline_data > T3_MAX_INLINE)
861 return ERR_PTR(-EINVAL);
862
863
864
865
866
867
868 sqsize = roundup_pow_of_two(attrs->cap.max_send_wr);
869 wqsize = roundup_pow_of_two(rqsize + sqsize);
870
871
872
873
874
875 ucontext = pd->uobject ? to_iwch_ucontext(pd->uobject->context) : NULL;
876 if (!ucontext && wqsize < (rqsize + (2 * sqsize)))
877 wqsize = roundup_pow_of_two(rqsize +
878 roundup_pow_of_two(attrs->cap.max_send_wr * 2));
879 pr_debug("%s wqsize %d sqsize %d rqsize %d\n", __func__,
880 wqsize, sqsize, rqsize);
881 qhp = kzalloc(sizeof(*qhp), GFP_KERNEL);
882 if (!qhp)
883 return ERR_PTR(-ENOMEM);
884 qhp->wq.size_log2 = ilog2(wqsize);
885 qhp->wq.rq_size_log2 = ilog2(rqsize);
886 qhp->wq.sq_size_log2 = ilog2(sqsize);
887 if (cxio_create_qp(&rhp->rdev, !udata, &qhp->wq,
888 ucontext ? &ucontext->uctx : &rhp->rdev.uctx)) {
889 kfree(qhp);
890 return ERR_PTR(-ENOMEM);
891 }
892
893 attrs->cap.max_recv_wr = rqsize - 1;
894 attrs->cap.max_send_wr = sqsize;
895 attrs->cap.max_inline_data = T3_MAX_INLINE;
896
897 qhp->rhp = rhp;
898 qhp->attr.pd = php->pdid;
899 qhp->attr.scq = ((struct iwch_cq *) attrs->send_cq)->cq.cqid;
900 qhp->attr.rcq = ((struct iwch_cq *) attrs->recv_cq)->cq.cqid;
901 qhp->attr.sq_num_entries = attrs->cap.max_send_wr;
902 qhp->attr.rq_num_entries = attrs->cap.max_recv_wr;
903 qhp->attr.sq_max_sges = attrs->cap.max_send_sge;
904 qhp->attr.sq_max_sges_rdma_write = attrs->cap.max_send_sge;
905 qhp->attr.rq_max_sges = attrs->cap.max_recv_sge;
906 qhp->attr.state = IWCH_QP_STATE_IDLE;
907 qhp->attr.next_state = IWCH_QP_STATE_IDLE;
908
909
910
911
912
913
914 qhp->attr.enable_rdma_read = 1;
915 qhp->attr.enable_rdma_write = 1;
916 qhp->attr.enable_bind = 1;
917 qhp->attr.max_ord = 1;
918 qhp->attr.max_ird = 1;
919
920 spin_lock_init(&qhp->lock);
921 init_waitqueue_head(&qhp->wait);
922 atomic_set(&qhp->refcnt, 1);
923
924 if (insert_handle(rhp, &rhp->qpidr, qhp, qhp->wq.qpid)) {
925 cxio_destroy_qp(&rhp->rdev, &qhp->wq,
926 ucontext ? &ucontext->uctx : &rhp->rdev.uctx);
927 kfree(qhp);
928 return ERR_PTR(-ENOMEM);
929 }
930
931 if (udata) {
932
933 struct iwch_mm_entry *mm1, *mm2;
934
935 mm1 = kmalloc(sizeof *mm1, GFP_KERNEL);
936 if (!mm1) {
937 iwch_destroy_qp(&qhp->ibqp);
938 return ERR_PTR(-ENOMEM);
939 }
940
941 mm2 = kmalloc(sizeof *mm2, GFP_KERNEL);
942 if (!mm2) {
943 kfree(mm1);
944 iwch_destroy_qp(&qhp->ibqp);
945 return ERR_PTR(-ENOMEM);
946 }
947
948 uresp.qpid = qhp->wq.qpid;
949 uresp.size_log2 = qhp->wq.size_log2;
950 uresp.sq_size_log2 = qhp->wq.sq_size_log2;
951 uresp.rq_size_log2 = qhp->wq.rq_size_log2;
952 spin_lock(&ucontext->mmap_lock);
953 uresp.key = ucontext->key;
954 ucontext->key += PAGE_SIZE;
955 uresp.db_key = ucontext->key;
956 ucontext->key += PAGE_SIZE;
957 spin_unlock(&ucontext->mmap_lock);
958 if (ib_copy_to_udata(udata, &uresp, sizeof (uresp))) {
959 kfree(mm1);
960 kfree(mm2);
961 iwch_destroy_qp(&qhp->ibqp);
962 return ERR_PTR(-EFAULT);
963 }
964 mm1->key = uresp.key;
965 mm1->addr = virt_to_phys(qhp->wq.queue);
966 mm1->len = PAGE_ALIGN(wqsize * sizeof (union t3_wr));
967 insert_mmap(ucontext, mm1);
968 mm2->key = uresp.db_key;
969 mm2->addr = qhp->wq.udb & PAGE_MASK;
970 mm2->len = PAGE_SIZE;
971 insert_mmap(ucontext, mm2);
972 }
973 qhp->ibqp.qp_num = qhp->wq.qpid;
974 pr_debug("%s sq_num_entries %d, rq_num_entries %d qpid 0x%0x qhp %p dma_addr 0x%llx size %d rq_addr 0x%x\n",
975 __func__, qhp->attr.sq_num_entries, qhp->attr.rq_num_entries,
976 qhp->wq.qpid, qhp, (unsigned long long)qhp->wq.dma_addr,
977 1 << qhp->wq.size_log2, qhp->wq.rq_addr);
978 return &qhp->ibqp;
979}
980
981static int iwch_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
982 int attr_mask, struct ib_udata *udata)
983{
984 struct iwch_dev *rhp;
985 struct iwch_qp *qhp;
986 enum iwch_qp_attr_mask mask = 0;
987 struct iwch_qp_attributes attrs;
988
989 pr_debug("%s ib_qp %p\n", __func__, ibqp);
990
991
992 if ((attr_mask & IB_QP_STATE) && (attr->qp_state == IB_QPS_RTR))
993 attr_mask &= ~IB_QP_STATE;
994
995
996 if (!attr_mask)
997 return 0;
998
999 memset(&attrs, 0, sizeof attrs);
1000 qhp = to_iwch_qp(ibqp);
1001 rhp = qhp->rhp;
1002
1003 attrs.next_state = iwch_convert_state(attr->qp_state);
1004 attrs.enable_rdma_read = (attr->qp_access_flags &
1005 IB_ACCESS_REMOTE_READ) ? 1 : 0;
1006 attrs.enable_rdma_write = (attr->qp_access_flags &
1007 IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
1008 attrs.enable_bind = (attr->qp_access_flags & IB_ACCESS_MW_BIND) ? 1 : 0;
1009
1010
1011 mask |= (attr_mask & IB_QP_STATE) ? IWCH_QP_ATTR_NEXT_STATE : 0;
1012 mask |= (attr_mask & IB_QP_ACCESS_FLAGS) ?
1013 (IWCH_QP_ATTR_ENABLE_RDMA_READ |
1014 IWCH_QP_ATTR_ENABLE_RDMA_WRITE |
1015 IWCH_QP_ATTR_ENABLE_RDMA_BIND) : 0;
1016
1017 return iwch_modify_qp(rhp, qhp, mask, &attrs, 0);
1018}
1019
1020void iwch_qp_add_ref(struct ib_qp *qp)
1021{
1022 pr_debug("%s ib_qp %p\n", __func__, qp);
1023 atomic_inc(&(to_iwch_qp(qp)->refcnt));
1024}
1025
1026void iwch_qp_rem_ref(struct ib_qp *qp)
1027{
1028 pr_debug("%s ib_qp %p\n", __func__, qp);
1029 if (atomic_dec_and_test(&(to_iwch_qp(qp)->refcnt)))
1030 wake_up(&(to_iwch_qp(qp)->wait));
1031}
1032
1033static struct ib_qp *iwch_get_qp(struct ib_device *dev, int qpn)
1034{
1035 pr_debug("%s ib_dev %p qpn 0x%x\n", __func__, dev, qpn);
1036 return (struct ib_qp *)get_qhp(to_iwch_dev(dev), qpn);
1037}
1038
1039
1040static int iwch_query_pkey(struct ib_device *ibdev,
1041 u8 port, u16 index, u16 * pkey)
1042{
1043 pr_debug("%s ibdev %p\n", __func__, ibdev);
1044 *pkey = 0;
1045 return 0;
1046}
1047
1048static int iwch_query_gid(struct ib_device *ibdev, u8 port,
1049 int index, union ib_gid *gid)
1050{
1051 struct iwch_dev *dev;
1052
1053 pr_debug("%s ibdev %p, port %d, index %d, gid %p\n",
1054 __func__, ibdev, port, index, gid);
1055 dev = to_iwch_dev(ibdev);
1056 BUG_ON(port == 0 || port > 2);
1057 memset(&(gid->raw[0]), 0, sizeof(gid->raw));
1058 memcpy(&(gid->raw[0]), dev->rdev.port_info.lldevs[port-1]->dev_addr, 6);
1059 return 0;
1060}
1061
1062static u64 fw_vers_string_to_u64(struct iwch_dev *iwch_dev)
1063{
1064 struct ethtool_drvinfo info;
1065 struct net_device *lldev = iwch_dev->rdev.t3cdev_p->lldev;
1066 char *cp, *next;
1067 unsigned fw_maj, fw_min, fw_mic;
1068
1069 lldev->ethtool_ops->get_drvinfo(lldev, &info);
1070
1071 next = info.fw_version + 1;
1072 cp = strsep(&next, ".");
1073 sscanf(cp, "%i", &fw_maj);
1074 cp = strsep(&next, ".");
1075 sscanf(cp, "%i", &fw_min);
1076 cp = strsep(&next, ".");
1077 sscanf(cp, "%i", &fw_mic);
1078
1079 return (((u64)fw_maj & 0xffff) << 32) | ((fw_min & 0xffff) << 16) |
1080 (fw_mic & 0xffff);
1081}
1082
1083static int iwch_query_device(struct ib_device *ibdev, struct ib_device_attr *props,
1084 struct ib_udata *uhw)
1085{
1086
1087 struct iwch_dev *dev;
1088
1089 pr_debug("%s ibdev %p\n", __func__, ibdev);
1090
1091 if (uhw->inlen || uhw->outlen)
1092 return -EINVAL;
1093
1094 dev = to_iwch_dev(ibdev);
1095 memset(props, 0, sizeof *props);
1096 memcpy(&props->sys_image_guid, dev->rdev.t3cdev_p->lldev->dev_addr, 6);
1097 props->hw_ver = dev->rdev.t3cdev_p->type;
1098 props->fw_ver = fw_vers_string_to_u64(dev);
1099 props->device_cap_flags = dev->device_cap_flags;
1100 props->page_size_cap = dev->attr.mem_pgsizes_bitmask;
1101 props->vendor_id = (u32)dev->rdev.rnic_info.pdev->vendor;
1102 props->vendor_part_id = (u32)dev->rdev.rnic_info.pdev->device;
1103 props->max_mr_size = dev->attr.max_mr_size;
1104 props->max_qp = dev->attr.max_qps;
1105 props->max_qp_wr = dev->attr.max_wrs;
1106 props->max_sge = dev->attr.max_sge_per_wr;
1107 props->max_sge_rd = 1;
1108 props->max_qp_rd_atom = dev->attr.max_rdma_reads_per_qp;
1109 props->max_qp_init_rd_atom = dev->attr.max_rdma_reads_per_qp;
1110 props->max_cq = dev->attr.max_cqs;
1111 props->max_cqe = dev->attr.max_cqes_per_cq;
1112 props->max_mr = dev->attr.max_mem_regs;
1113 props->max_pd = dev->attr.max_pds;
1114 props->local_ca_ack_delay = 0;
1115 props->max_fast_reg_page_list_len = T3_MAX_FASTREG_DEPTH;
1116
1117 return 0;
1118}
1119
1120static int iwch_query_port(struct ib_device *ibdev,
1121 u8 port, struct ib_port_attr *props)
1122{
1123 struct iwch_dev *dev;
1124 struct net_device *netdev;
1125 struct in_device *inetdev;
1126
1127 pr_debug("%s ibdev %p\n", __func__, ibdev);
1128
1129 dev = to_iwch_dev(ibdev);
1130 netdev = dev->rdev.port_info.lldevs[port-1];
1131
1132
1133 props->max_mtu = IB_MTU_4096;
1134 props->active_mtu = ib_mtu_int_to_enum(netdev->mtu);
1135
1136 if (!netif_carrier_ok(netdev))
1137 props->state = IB_PORT_DOWN;
1138 else {
1139 inetdev = in_dev_get(netdev);
1140 if (inetdev) {
1141 if (inetdev->ifa_list)
1142 props->state = IB_PORT_ACTIVE;
1143 else
1144 props->state = IB_PORT_INIT;
1145 in_dev_put(inetdev);
1146 } else
1147 props->state = IB_PORT_INIT;
1148 }
1149
1150 props->port_cap_flags =
1151 IB_PORT_CM_SUP |
1152 IB_PORT_SNMP_TUNNEL_SUP |
1153 IB_PORT_REINIT_SUP |
1154 IB_PORT_DEVICE_MGMT_SUP |
1155 IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP;
1156 props->gid_tbl_len = 1;
1157 props->pkey_tbl_len = 1;
1158 props->active_width = 2;
1159 props->active_speed = IB_SPEED_DDR;
1160 props->max_msg_sz = -1;
1161
1162 return 0;
1163}
1164
1165static ssize_t show_rev(struct device *dev, struct device_attribute *attr,
1166 char *buf)
1167{
1168 struct iwch_dev *iwch_dev = container_of(dev, struct iwch_dev,
1169 ibdev.dev);
1170 pr_debug("%s dev 0x%p\n", __func__, dev);
1171 return sprintf(buf, "%d\n", iwch_dev->rdev.t3cdev_p->type);
1172}
1173
1174static ssize_t show_hca(struct device *dev, struct device_attribute *attr,
1175 char *buf)
1176{
1177 struct iwch_dev *iwch_dev = container_of(dev, struct iwch_dev,
1178 ibdev.dev);
1179 struct ethtool_drvinfo info;
1180 struct net_device *lldev = iwch_dev->rdev.t3cdev_p->lldev;
1181
1182 pr_debug("%s dev 0x%p\n", __func__, dev);
1183 lldev->ethtool_ops->get_drvinfo(lldev, &info);
1184 return sprintf(buf, "%s\n", info.driver);
1185}
1186
1187static ssize_t show_board(struct device *dev, struct device_attribute *attr,
1188 char *buf)
1189{
1190 struct iwch_dev *iwch_dev = container_of(dev, struct iwch_dev,
1191 ibdev.dev);
1192 pr_debug("%s dev 0x%p\n", __func__, dev);
1193 return sprintf(buf, "%x.%x\n", iwch_dev->rdev.rnic_info.pdev->vendor,
1194 iwch_dev->rdev.rnic_info.pdev->device);
1195}
1196
1197enum counters {
1198 IPINRECEIVES,
1199 IPINHDRERRORS,
1200 IPINADDRERRORS,
1201 IPINUNKNOWNPROTOS,
1202 IPINDISCARDS,
1203 IPINDELIVERS,
1204 IPOUTREQUESTS,
1205 IPOUTDISCARDS,
1206 IPOUTNOROUTES,
1207 IPREASMTIMEOUT,
1208 IPREASMREQDS,
1209 IPREASMOKS,
1210 IPREASMFAILS,
1211 TCPACTIVEOPENS,
1212 TCPPASSIVEOPENS,
1213 TCPATTEMPTFAILS,
1214 TCPESTABRESETS,
1215 TCPCURRESTAB,
1216 TCPINSEGS,
1217 TCPOUTSEGS,
1218 TCPRETRANSSEGS,
1219 TCPINERRS,
1220 TCPOUTRSTS,
1221 TCPRTOMIN,
1222 TCPRTOMAX,
1223 NR_COUNTERS
1224};
1225
1226static const char * const names[] = {
1227 [IPINRECEIVES] = "ipInReceives",
1228 [IPINHDRERRORS] = "ipInHdrErrors",
1229 [IPINADDRERRORS] = "ipInAddrErrors",
1230 [IPINUNKNOWNPROTOS] = "ipInUnknownProtos",
1231 [IPINDISCARDS] = "ipInDiscards",
1232 [IPINDELIVERS] = "ipInDelivers",
1233 [IPOUTREQUESTS] = "ipOutRequests",
1234 [IPOUTDISCARDS] = "ipOutDiscards",
1235 [IPOUTNOROUTES] = "ipOutNoRoutes",
1236 [IPREASMTIMEOUT] = "ipReasmTimeout",
1237 [IPREASMREQDS] = "ipReasmReqds",
1238 [IPREASMOKS] = "ipReasmOKs",
1239 [IPREASMFAILS] = "ipReasmFails",
1240 [TCPACTIVEOPENS] = "tcpActiveOpens",
1241 [TCPPASSIVEOPENS] = "tcpPassiveOpens",
1242 [TCPATTEMPTFAILS] = "tcpAttemptFails",
1243 [TCPESTABRESETS] = "tcpEstabResets",
1244 [TCPCURRESTAB] = "tcpCurrEstab",
1245 [TCPINSEGS] = "tcpInSegs",
1246 [TCPOUTSEGS] = "tcpOutSegs",
1247 [TCPRETRANSSEGS] = "tcpRetransSegs",
1248 [TCPINERRS] = "tcpInErrs",
1249 [TCPOUTRSTS] = "tcpOutRsts",
1250 [TCPRTOMIN] = "tcpRtoMin",
1251 [TCPRTOMAX] = "tcpRtoMax",
1252};
1253
1254static struct rdma_hw_stats *iwch_alloc_stats(struct ib_device *ibdev,
1255 u8 port_num)
1256{
1257 BUILD_BUG_ON(ARRAY_SIZE(names) != NR_COUNTERS);
1258
1259
1260 if (port_num != 0)
1261 return NULL;
1262
1263 return rdma_alloc_hw_stats_struct(names, NR_COUNTERS,
1264 RDMA_HW_STATS_DEFAULT_LIFESPAN);
1265}
1266
1267static int iwch_get_mib(struct ib_device *ibdev, struct rdma_hw_stats *stats,
1268 u8 port, int index)
1269{
1270 struct iwch_dev *dev;
1271 struct tp_mib_stats m;
1272 int ret;
1273
1274 if (port != 0 || !stats)
1275 return -ENOSYS;
1276
1277 pr_debug("%s ibdev %p\n", __func__, ibdev);
1278 dev = to_iwch_dev(ibdev);
1279 ret = dev->rdev.t3cdev_p->ctl(dev->rdev.t3cdev_p, RDMA_GET_MIB, &m);
1280 if (ret)
1281 return -ENOSYS;
1282
1283 stats->value[IPINRECEIVES] = ((u64)m.ipInReceive_hi << 32) + m.ipInReceive_lo;
1284 stats->value[IPINHDRERRORS] = ((u64)m.ipInHdrErrors_hi << 32) + m.ipInHdrErrors_lo;
1285 stats->value[IPINADDRERRORS] = ((u64)m.ipInAddrErrors_hi << 32) + m.ipInAddrErrors_lo;
1286 stats->value[IPINUNKNOWNPROTOS] = ((u64)m.ipInUnknownProtos_hi << 32) + m.ipInUnknownProtos_lo;
1287 stats->value[IPINDISCARDS] = ((u64)m.ipInDiscards_hi << 32) + m.ipInDiscards_lo;
1288 stats->value[IPINDELIVERS] = ((u64)m.ipInDelivers_hi << 32) + m.ipInDelivers_lo;
1289 stats->value[IPOUTREQUESTS] = ((u64)m.ipOutRequests_hi << 32) + m.ipOutRequests_lo;
1290 stats->value[IPOUTDISCARDS] = ((u64)m.ipOutDiscards_hi << 32) + m.ipOutDiscards_lo;
1291 stats->value[IPOUTNOROUTES] = ((u64)m.ipOutNoRoutes_hi << 32) + m.ipOutNoRoutes_lo;
1292 stats->value[IPREASMTIMEOUT] = m.ipReasmTimeout;
1293 stats->value[IPREASMREQDS] = m.ipReasmReqds;
1294 stats->value[IPREASMOKS] = m.ipReasmOKs;
1295 stats->value[IPREASMFAILS] = m.ipReasmFails;
1296 stats->value[TCPACTIVEOPENS] = m.tcpActiveOpens;
1297 stats->value[TCPPASSIVEOPENS] = m.tcpPassiveOpens;
1298 stats->value[TCPATTEMPTFAILS] = m.tcpAttemptFails;
1299 stats->value[TCPESTABRESETS] = m.tcpEstabResets;
1300 stats->value[TCPCURRESTAB] = m.tcpOutRsts;
1301 stats->value[TCPINSEGS] = m.tcpCurrEstab;
1302 stats->value[TCPOUTSEGS] = ((u64)m.tcpInSegs_hi << 32) + m.tcpInSegs_lo;
1303 stats->value[TCPRETRANSSEGS] = ((u64)m.tcpOutSegs_hi << 32) + m.tcpOutSegs_lo;
1304 stats->value[TCPINERRS] = ((u64)m.tcpRetransSeg_hi << 32) + m.tcpRetransSeg_lo,
1305 stats->value[TCPOUTRSTS] = ((u64)m.tcpInErrs_hi << 32) + m.tcpInErrs_lo;
1306 stats->value[TCPRTOMIN] = m.tcpRtoMin;
1307 stats->value[TCPRTOMAX] = m.tcpRtoMax;
1308
1309 return stats->num_counters;
1310}
1311
1312static DEVICE_ATTR(hw_rev, S_IRUGO, show_rev, NULL);
1313static DEVICE_ATTR(hca_type, S_IRUGO, show_hca, NULL);
1314static DEVICE_ATTR(board_id, S_IRUGO, show_board, NULL);
1315
1316static struct device_attribute *iwch_class_attributes[] = {
1317 &dev_attr_hw_rev,
1318 &dev_attr_hca_type,
1319 &dev_attr_board_id,
1320};
1321
1322static int iwch_port_immutable(struct ib_device *ibdev, u8 port_num,
1323 struct ib_port_immutable *immutable)
1324{
1325 struct ib_port_attr attr;
1326 int err;
1327
1328 immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
1329
1330 err = ib_query_port(ibdev, port_num, &attr);
1331 if (err)
1332 return err;
1333
1334 immutable->pkey_tbl_len = attr.pkey_tbl_len;
1335 immutable->gid_tbl_len = attr.gid_tbl_len;
1336
1337 return 0;
1338}
1339
1340static void get_dev_fw_ver_str(struct ib_device *ibdev, char *str)
1341{
1342 struct iwch_dev *iwch_dev = to_iwch_dev(ibdev);
1343 struct ethtool_drvinfo info;
1344 struct net_device *lldev = iwch_dev->rdev.t3cdev_p->lldev;
1345
1346 pr_debug("%s dev 0x%p\n", __func__, iwch_dev);
1347 lldev->ethtool_ops->get_drvinfo(lldev, &info);
1348 snprintf(str, IB_FW_VERSION_NAME_MAX, "%s", info.fw_version);
1349}
1350
1351int iwch_register_device(struct iwch_dev *dev)
1352{
1353 int ret;
1354 int i;
1355
1356 pr_debug("%s iwch_dev %p\n", __func__, dev);
1357 strlcpy(dev->ibdev.name, "cxgb3_%d", IB_DEVICE_NAME_MAX);
1358 memset(&dev->ibdev.node_guid, 0, sizeof(dev->ibdev.node_guid));
1359 memcpy(&dev->ibdev.node_guid, dev->rdev.t3cdev_p->lldev->dev_addr, 6);
1360 dev->ibdev.owner = THIS_MODULE;
1361 dev->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY |
1362 IB_DEVICE_MEM_WINDOW |
1363 IB_DEVICE_MEM_MGT_EXTENSIONS;
1364
1365
1366 dev->ibdev.local_dma_lkey = 0;
1367
1368 dev->ibdev.uverbs_cmd_mask =
1369 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
1370 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
1371 (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
1372 (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
1373 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
1374 (1ull << IB_USER_VERBS_CMD_REG_MR) |
1375 (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
1376 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
1377 (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
1378 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
1379 (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) |
1380 (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
1381 (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
1382 (1ull << IB_USER_VERBS_CMD_POLL_CQ) |
1383 (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
1384 (1ull << IB_USER_VERBS_CMD_POST_SEND) |
1385 (1ull << IB_USER_VERBS_CMD_POST_RECV);
1386 dev->ibdev.node_type = RDMA_NODE_RNIC;
1387 BUILD_BUG_ON(sizeof(IWCH_NODE_DESC) > IB_DEVICE_NODE_DESC_MAX);
1388 memcpy(dev->ibdev.node_desc, IWCH_NODE_DESC, sizeof(IWCH_NODE_DESC));
1389 dev->ibdev.phys_port_cnt = dev->rdev.port_info.nports;
1390 dev->ibdev.num_comp_vectors = 1;
1391 dev->ibdev.dev.parent = &dev->rdev.rnic_info.pdev->dev;
1392 dev->ibdev.query_device = iwch_query_device;
1393 dev->ibdev.query_port = iwch_query_port;
1394 dev->ibdev.query_pkey = iwch_query_pkey;
1395 dev->ibdev.query_gid = iwch_query_gid;
1396 dev->ibdev.alloc_ucontext = iwch_alloc_ucontext;
1397 dev->ibdev.dealloc_ucontext = iwch_dealloc_ucontext;
1398 dev->ibdev.mmap = iwch_mmap;
1399 dev->ibdev.alloc_pd = iwch_allocate_pd;
1400 dev->ibdev.dealloc_pd = iwch_deallocate_pd;
1401 dev->ibdev.create_ah = iwch_ah_create;
1402 dev->ibdev.destroy_ah = iwch_ah_destroy;
1403 dev->ibdev.create_qp = iwch_create_qp;
1404 dev->ibdev.modify_qp = iwch_ib_modify_qp;
1405 dev->ibdev.destroy_qp = iwch_destroy_qp;
1406 dev->ibdev.create_cq = iwch_create_cq;
1407 dev->ibdev.destroy_cq = iwch_destroy_cq;
1408 dev->ibdev.resize_cq = iwch_resize_cq;
1409 dev->ibdev.poll_cq = iwch_poll_cq;
1410 dev->ibdev.get_dma_mr = iwch_get_dma_mr;
1411 dev->ibdev.reg_user_mr = iwch_reg_user_mr;
1412 dev->ibdev.dereg_mr = iwch_dereg_mr;
1413 dev->ibdev.alloc_mw = iwch_alloc_mw;
1414 dev->ibdev.dealloc_mw = iwch_dealloc_mw;
1415 dev->ibdev.alloc_mr = iwch_alloc_mr;
1416 dev->ibdev.map_mr_sg = iwch_map_mr_sg;
1417 dev->ibdev.attach_mcast = iwch_multicast_attach;
1418 dev->ibdev.detach_mcast = iwch_multicast_detach;
1419 dev->ibdev.process_mad = iwch_process_mad;
1420 dev->ibdev.req_notify_cq = iwch_arm_cq;
1421 dev->ibdev.post_send = iwch_post_send;
1422 dev->ibdev.post_recv = iwch_post_receive;
1423 dev->ibdev.alloc_hw_stats = iwch_alloc_stats;
1424 dev->ibdev.get_hw_stats = iwch_get_mib;
1425 dev->ibdev.uverbs_abi_ver = IWCH_UVERBS_ABI_VERSION;
1426 dev->ibdev.get_port_immutable = iwch_port_immutable;
1427 dev->ibdev.get_dev_fw_str = get_dev_fw_ver_str;
1428
1429 dev->ibdev.iwcm = kmalloc(sizeof(struct iw_cm_verbs), GFP_KERNEL);
1430 if (!dev->ibdev.iwcm)
1431 return -ENOMEM;
1432
1433 dev->ibdev.iwcm->connect = iwch_connect;
1434 dev->ibdev.iwcm->accept = iwch_accept_cr;
1435 dev->ibdev.iwcm->reject = iwch_reject_cr;
1436 dev->ibdev.iwcm->create_listen = iwch_create_listen;
1437 dev->ibdev.iwcm->destroy_listen = iwch_destroy_listen;
1438 dev->ibdev.iwcm->add_ref = iwch_qp_add_ref;
1439 dev->ibdev.iwcm->rem_ref = iwch_qp_rem_ref;
1440 dev->ibdev.iwcm->get_qp = iwch_get_qp;
1441 memcpy(dev->ibdev.iwcm->ifname, dev->rdev.t3cdev_p->lldev->name,
1442 sizeof(dev->ibdev.iwcm->ifname));
1443
1444 dev->ibdev.driver_id = RDMA_DRIVER_CXGB3;
1445 ret = ib_register_device(&dev->ibdev, NULL);
1446 if (ret)
1447 goto bail1;
1448
1449 for (i = 0; i < ARRAY_SIZE(iwch_class_attributes); ++i) {
1450 ret = device_create_file(&dev->ibdev.dev,
1451 iwch_class_attributes[i]);
1452 if (ret) {
1453 goto bail2;
1454 }
1455 }
1456 return 0;
1457bail2:
1458 ib_unregister_device(&dev->ibdev);
1459bail1:
1460 kfree(dev->ibdev.iwcm);
1461 return ret;
1462}
1463
1464void iwch_unregister_device(struct iwch_dev *dev)
1465{
1466 int i;
1467
1468 pr_debug("%s iwch_dev %p\n", __func__, dev);
1469 for (i = 0; i < ARRAY_SIZE(iwch_class_attributes); ++i)
1470 device_remove_file(&dev->ibdev.dev,
1471 iwch_class_attributes[i]);
1472 ib_unregister_device(&dev->ibdev);
1473 kfree(dev->ibdev.iwcm);
1474 return;
1475}
1476
