1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36#include <linux/module.h>
37#include <linux/moduleparam.h>
38#include <linux/pci.h>
39#include <linux/netdevice.h>
40#include <linux/etherdevice.h>
41#include <linux/delay.h>
42#include <linux/ethtool.h>
43#include <linux/mii.h>
44#include <linux/if_vlan.h>
45#include <linux/crc32.h>
46#include <linux/in.h>
47#include <linux/ip.h>
48#include <linux/tcp.h>
49#include <linux/init.h>
50#include <linux/dma-mapping.h>
51#include <linux/mm.h>
52#include <linux/inet.h>
53#include <linux/vmalloc.h>
54#include <linux/slab.h>
55
56#include <linux/route.h>
57
58#include <asm/io.h>
59#include <asm/irq.h>
60#include <asm/byteorder.h>
61#include <rdma/ib_smi.h>
62#include "c2.h"
63#include "c2_vq.h"
64
65
66#define C2_MIN_PAGESIZE 1024
67
68#define C2_MAX_MRS 32768
69#define C2_MAX_QPS 16000
70#define C2_MAX_WQE_SZ 256
71#define C2_MAX_QP_WR ((128*1024)/C2_MAX_WQE_SZ)
72#define C2_MAX_SGES 4
73#define C2_MAX_SGE_RD 1
74#define C2_MAX_CQS 32768
75#define C2_MAX_CQES 4096
76#define C2_MAX_PDS 16384
77
78
79
80
81static int c2_adapter_init(struct c2_dev *c2dev)
82{
83 struct c2wr_init_req wr;
84 int err;
85
86 memset(&wr, 0, sizeof(wr));
87 c2_wr_set_id(&wr, CCWR_INIT);
88 wr.hdr.context = 0;
89 wr.hint_count = cpu_to_be64(c2dev->hint_count_dma);
90 wr.q0_host_shared = cpu_to_be64(c2dev->req_vq.shared_dma);
91 wr.q1_host_shared = cpu_to_be64(c2dev->rep_vq.shared_dma);
92 wr.q1_host_msg_pool = cpu_to_be64(c2dev->rep_vq.host_dma);
93 wr.q2_host_shared = cpu_to_be64(c2dev->aeq.shared_dma);
94 wr.q2_host_msg_pool = cpu_to_be64(c2dev->aeq.host_dma);
95
96
97 err = vq_send_wr(c2dev, (union c2wr *) & wr);
98
99 return err;
100}
101
102
103
104
105static void c2_adapter_term(struct c2_dev *c2dev)
106{
107 struct c2wr_init_req wr;
108
109 memset(&wr, 0, sizeof(wr));
110 c2_wr_set_id(&wr, CCWR_TERM);
111 wr.hdr.context = 0;
112
113
114 vq_send_wr(c2dev, (union c2wr *) & wr);
115 c2dev->init = 0;
116
117 return;
118}
119
120
121
122
123static int c2_rnic_query(struct c2_dev *c2dev, struct ib_device_attr *props)
124{
125 struct c2_vq_req *vq_req;
126 struct c2wr_rnic_query_req wr;
127 struct c2wr_rnic_query_rep *reply;
128 int err;
129
130 vq_req = vq_req_alloc(c2dev);
131 if (!vq_req)
132 return -ENOMEM;
133
134 c2_wr_set_id(&wr, CCWR_RNIC_QUERY);
135 wr.hdr.context = (unsigned long) vq_req;
136 wr.rnic_handle = c2dev->adapter_handle;
137
138 vq_req_get(c2dev, vq_req);
139
140 err = vq_send_wr(c2dev, (union c2wr *) &wr);
141 if (err) {
142 vq_req_put(c2dev, vq_req);
143 goto bail1;
144 }
145
146 err = vq_wait_for_reply(c2dev, vq_req);
147 if (err)
148 goto bail1;
149
150 reply =
151 (struct c2wr_rnic_query_rep *) (unsigned long) (vq_req->reply_msg);
152 if (!reply)
153 err = -ENOMEM;
154 else
155 err = c2_errno(reply);
156 if (err)
157 goto bail2;
158
159 props->fw_ver =
160 ((u64)be32_to_cpu(reply->fw_ver_major) << 32) |
161 ((be32_to_cpu(reply->fw_ver_minor) & 0xFFFF) << 16) |
162 (be32_to_cpu(reply->fw_ver_patch) & 0xFFFF);
163 memcpy(&props->sys_image_guid, c2dev->netdev->dev_addr, 6);
164 props->max_mr_size = 0xFFFFFFFF;
165 props->page_size_cap = ~(C2_MIN_PAGESIZE-1);
166 props->vendor_id = be32_to_cpu(reply->vendor_id);
167 props->vendor_part_id = be32_to_cpu(reply->part_number);
168 props->hw_ver = be32_to_cpu(reply->hw_version);
169 props->max_qp = be32_to_cpu(reply->max_qps);
170 props->max_qp_wr = be32_to_cpu(reply->max_qp_depth);
171 props->device_cap_flags = c2dev->device_cap_flags;
172 props->max_sge = C2_MAX_SGES;
173 props->max_sge_rd = C2_MAX_SGE_RD;
174 props->max_cq = be32_to_cpu(reply->max_cqs);
175 props->max_cqe = be32_to_cpu(reply->max_cq_depth);
176 props->max_mr = be32_to_cpu(reply->max_mrs);
177 props->max_pd = be32_to_cpu(reply->max_pds);
178 props->max_qp_rd_atom = be32_to_cpu(reply->max_qp_ird);
179 props->max_ee_rd_atom = 0;
180 props->max_res_rd_atom = be32_to_cpu(reply->max_global_ird);
181 props->max_qp_init_rd_atom = be32_to_cpu(reply->max_qp_ord);
182 props->max_ee_init_rd_atom = 0;
183 props->atomic_cap = IB_ATOMIC_NONE;
184 props->max_ee = 0;
185 props->max_rdd = 0;
186 props->max_mw = be32_to_cpu(reply->max_mws);
187 props->max_raw_ipv6_qp = 0;
188 props->max_raw_ethy_qp = 0;
189 props->max_mcast_grp = 0;
190 props->max_mcast_qp_attach = 0;
191 props->max_total_mcast_qp_attach = 0;
192 props->max_ah = 0;
193 props->max_fmr = 0;
194 props->max_map_per_fmr = 0;
195 props->max_srq = 0;
196 props->max_srq_wr = 0;
197 props->max_srq_sge = 0;
198 props->max_pkeys = 0;
199 props->local_ca_ack_delay = 0;
200
201 bail2:
202 vq_repbuf_free(c2dev, reply);
203
204 bail1:
205 vq_req_free(c2dev, vq_req);
206 return err;
207}
208
209
210
211
212int c2_add_addr(struct c2_dev *c2dev, __be32 inaddr, __be32 inmask)
213{
214 struct c2_vq_req *vq_req;
215 struct c2wr_rnic_setconfig_req *wr;
216 struct c2wr_rnic_setconfig_rep *reply;
217 struct c2_netaddr netaddr;
218 int err, len;
219
220 vq_req = vq_req_alloc(c2dev);
221 if (!vq_req)
222 return -ENOMEM;
223
224 len = sizeof(struct c2_netaddr);
225 wr = kmalloc(c2dev->req_vq.msg_size, GFP_KERNEL);
226 if (!wr) {
227 err = -ENOMEM;
228 goto bail0;
229 }
230
231 c2_wr_set_id(wr, CCWR_RNIC_SETCONFIG);
232 wr->hdr.context = (unsigned long) vq_req;
233 wr->rnic_handle = c2dev->adapter_handle;
234 wr->option = cpu_to_be32(C2_CFG_ADD_ADDR);
235
236 netaddr.ip_addr = inaddr;
237 netaddr.netmask = inmask;
238 netaddr.mtu = 0;
239
240 memcpy(wr->data, &netaddr, len);
241
242 vq_req_get(c2dev, vq_req);
243
244 err = vq_send_wr(c2dev, (union c2wr *) wr);
245 if (err) {
246 vq_req_put(c2dev, vq_req);
247 goto bail1;
248 }
249
250 err = vq_wait_for_reply(c2dev, vq_req);
251 if (err)
252 goto bail1;
253
254 reply =
255 (struct c2wr_rnic_setconfig_rep *) (unsigned long) (vq_req->reply_msg);
256 if (!reply) {
257 err = -ENOMEM;
258 goto bail1;
259 }
260
261 err = c2_errno(reply);
262 vq_repbuf_free(c2dev, reply);
263
264 bail1:
265 kfree(wr);
266 bail0:
267 vq_req_free(c2dev, vq_req);
268 return err;
269}
270
271
272
273
274int c2_del_addr(struct c2_dev *c2dev, __be32 inaddr, __be32 inmask)
275{
276 struct c2_vq_req *vq_req;
277 struct c2wr_rnic_setconfig_req *wr;
278 struct c2wr_rnic_setconfig_rep *reply;
279 struct c2_netaddr netaddr;
280 int err, len;
281
282 vq_req = vq_req_alloc(c2dev);
283 if (!vq_req)
284 return -ENOMEM;
285
286 len = sizeof(struct c2_netaddr);
287 wr = kmalloc(c2dev->req_vq.msg_size, GFP_KERNEL);
288 if (!wr) {
289 err = -ENOMEM;
290 goto bail0;
291 }
292
293 c2_wr_set_id(wr, CCWR_RNIC_SETCONFIG);
294 wr->hdr.context = (unsigned long) vq_req;
295 wr->rnic_handle = c2dev->adapter_handle;
296 wr->option = cpu_to_be32(C2_CFG_DEL_ADDR);
297
298 netaddr.ip_addr = inaddr;
299 netaddr.netmask = inmask;
300 netaddr.mtu = 0;
301
302 memcpy(wr->data, &netaddr, len);
303
304 vq_req_get(c2dev, vq_req);
305
306 err = vq_send_wr(c2dev, (union c2wr *) wr);
307 if (err) {
308 vq_req_put(c2dev, vq_req);
309 goto bail1;
310 }
311
312 err = vq_wait_for_reply(c2dev, vq_req);
313 if (err)
314 goto bail1;
315
316 reply =
317 (struct c2wr_rnic_setconfig_rep *) (unsigned long) (vq_req->reply_msg);
318 if (!reply) {
319 err = -ENOMEM;
320 goto bail1;
321 }
322
323 err = c2_errno(reply);
324 vq_repbuf_free(c2dev, reply);
325
326 bail1:
327 kfree(wr);
328 bail0:
329 vq_req_free(c2dev, vq_req);
330 return err;
331}
332
333
334
335
336
337static int c2_rnic_open(struct c2_dev *c2dev)
338{
339 struct c2_vq_req *vq_req;
340 union c2wr wr;
341 struct c2wr_rnic_open_rep *reply;
342 int err;
343
344 vq_req = vq_req_alloc(c2dev);
345 if (vq_req == NULL) {
346 return -ENOMEM;
347 }
348
349 memset(&wr, 0, sizeof(wr));
350 c2_wr_set_id(&wr, CCWR_RNIC_OPEN);
351 wr.rnic_open.req.hdr.context = (unsigned long) (vq_req);
352 wr.rnic_open.req.flags = cpu_to_be16(RNIC_PRIV_MODE);
353 wr.rnic_open.req.port_num = cpu_to_be16(0);
354 wr.rnic_open.req.user_context = (unsigned long) c2dev;
355
356 vq_req_get(c2dev, vq_req);
357
358 err = vq_send_wr(c2dev, &wr);
359 if (err) {
360 vq_req_put(c2dev, vq_req);
361 goto bail0;
362 }
363
364 err = vq_wait_for_reply(c2dev, vq_req);
365 if (err) {
366 goto bail0;
367 }
368
369 reply = (struct c2wr_rnic_open_rep *) (unsigned long) (vq_req->reply_msg);
370 if (!reply) {
371 err = -ENOMEM;
372 goto bail0;
373 }
374
375 if ((err = c2_errno(reply)) != 0) {
376 goto bail1;
377 }
378
379 c2dev->adapter_handle = reply->rnic_handle;
380
381 bail1:
382 vq_repbuf_free(c2dev, reply);
383 bail0:
384 vq_req_free(c2dev, vq_req);
385 return err;
386}
387
388
389
390
391static int c2_rnic_close(struct c2_dev *c2dev)
392{
393 struct c2_vq_req *vq_req;
394 union c2wr wr;
395 struct c2wr_rnic_close_rep *reply;
396 int err;
397
398 vq_req = vq_req_alloc(c2dev);
399 if (vq_req == NULL) {
400 return -ENOMEM;
401 }
402
403 memset(&wr, 0, sizeof(wr));
404 c2_wr_set_id(&wr, CCWR_RNIC_CLOSE);
405 wr.rnic_close.req.hdr.context = (unsigned long) vq_req;
406 wr.rnic_close.req.rnic_handle = c2dev->adapter_handle;
407
408 vq_req_get(c2dev, vq_req);
409
410 err = vq_send_wr(c2dev, &wr);
411 if (err) {
412 vq_req_put(c2dev, vq_req);
413 goto bail0;
414 }
415
416 err = vq_wait_for_reply(c2dev, vq_req);
417 if (err) {
418 goto bail0;
419 }
420
421 reply = (struct c2wr_rnic_close_rep *) (unsigned long) (vq_req->reply_msg);
422 if (!reply) {
423 err = -ENOMEM;
424 goto bail0;
425 }
426
427 if ((err = c2_errno(reply)) != 0) {
428 goto bail1;
429 }
430
431 c2dev->adapter_handle = 0;
432
433 bail1:
434 vq_repbuf_free(c2dev, reply);
435 bail0:
436 vq_req_free(c2dev, vq_req);
437 return err;
438}
439
440
441
442
443
444
445int c2_rnic_init(struct c2_dev *c2dev)
446{
447 int err;
448 u32 qsize, msgsize;
449 void *q1_pages;
450 void *q2_pages;
451 void __iomem *mmio_regs;
452
453
454 c2dev->device_cap_flags =
455 (IB_DEVICE_RESIZE_MAX_WR |
456 IB_DEVICE_CURR_QP_STATE_MOD |
457 IB_DEVICE_SYS_IMAGE_GUID |
458 IB_DEVICE_LOCAL_DMA_LKEY |
459 IB_DEVICE_MEM_WINDOW);
460
461
462 c2dev->qptr_array = vzalloc(C2_MAX_CQS * sizeof(void *));
463 if (!c2dev->qptr_array) {
464 return -ENOMEM;
465 }
466
467
468 c2dev->qptr_array[0] = (void *) &c2dev->req_vq;
469 c2dev->qptr_array[1] = (void *) &c2dev->rep_vq;
470 c2dev->qptr_array[2] = (void *) &c2dev->aeq;
471
472
473 init_waitqueue_head(&c2dev->req_vq_wo);
474 spin_lock_init(&c2dev->vqlock);
475 spin_lock_init(&c2dev->lock);
476
477
478
479
480 err = c2_init_mqsp_pool(c2dev, GFP_KERNEL, &c2dev->kern_mqsp_pool);
481 if (err) {
482 goto bail0;
483 }
484
485
486
487
488
489 c2dev->hint_count = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
490 &c2dev->hint_count_dma,
491 GFP_KERNEL);
492 c2dev->req_vq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
493 &c2dev->req_vq.shared_dma,
494 GFP_KERNEL);
495 c2dev->rep_vq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
496 &c2dev->rep_vq.shared_dma,
497 GFP_KERNEL);
498 c2dev->aeq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
499 &c2dev->aeq.shared_dma, GFP_KERNEL);
500 if (!c2dev->hint_count || !c2dev->req_vq.shared ||
501 !c2dev->rep_vq.shared || !c2dev->aeq.shared) {
502 err = -ENOMEM;
503 goto bail1;
504 }
505
506 mmio_regs = c2dev->kva;
507
508 c2_mq_req_init(&c2dev->req_vq, 0,
509 be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q0_QSIZE)),
510 be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q0_MSGSIZE)),
511 mmio_regs +
512 be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q0_POOLSTART)),
513 mmio_regs +
514 be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q0_SHARED)),
515 C2_MQ_ADAPTER_TARGET);
516
517
518 qsize = be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q1_QSIZE));
519 msgsize = be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q1_MSGSIZE));
520 q1_pages = dma_alloc_coherent(&c2dev->pcidev->dev, qsize * msgsize,
521 &c2dev->rep_vq.host_dma, GFP_KERNEL);
522 if (!q1_pages) {
523 err = -ENOMEM;
524 goto bail1;
525 }
526 dma_unmap_addr_set(&c2dev->rep_vq, mapping, c2dev->rep_vq.host_dma);
527 pr_debug("%s rep_vq va %p dma %llx\n", __func__, q1_pages,
528 (unsigned long long) c2dev->rep_vq.host_dma);
529 c2_mq_rep_init(&c2dev->rep_vq,
530 1,
531 qsize,
532 msgsize,
533 q1_pages,
534 mmio_regs +
535 be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q1_SHARED)),
536 C2_MQ_HOST_TARGET);
537
538
539 qsize = be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q2_QSIZE));
540 msgsize = be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q2_MSGSIZE));
541 q2_pages = dma_alloc_coherent(&c2dev->pcidev->dev, qsize * msgsize,
542 &c2dev->aeq.host_dma, GFP_KERNEL);
543 if (!q2_pages) {
544 err = -ENOMEM;
545 goto bail2;
546 }
547 dma_unmap_addr_set(&c2dev->aeq, mapping, c2dev->aeq.host_dma);
548 pr_debug("%s aeq va %p dma %llx\n", __func__, q2_pages,
549 (unsigned long long) c2dev->aeq.host_dma);
550 c2_mq_rep_init(&c2dev->aeq,
551 2,
552 qsize,
553 msgsize,
554 q2_pages,
555 mmio_regs +
556 be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q2_SHARED)),
557 C2_MQ_HOST_TARGET);
558
559
560 err = vq_init(c2dev);
561 if (err)
562 goto bail3;
563
564
565 writel(0, c2dev->regs + C2_IDIS);
566
567
568 err = c2_adapter_init(c2dev);
569 if (err)
570 goto bail4;
571 c2dev->init++;
572
573
574 err = c2_rnic_open(c2dev);
575 if (err)
576 goto bail4;
577
578
579 if (c2_rnic_query(c2dev, &c2dev->props))
580 goto bail5;
581
582
583 err = c2_init_pd_table(c2dev);
584 if (err)
585 goto bail5;
586
587
588 c2_init_qp_table(c2dev);
589 return 0;
590
591 bail5:
592 c2_rnic_close(c2dev);
593 bail4:
594 vq_term(c2dev);
595 bail3:
596 dma_free_coherent(&c2dev->pcidev->dev,
597 c2dev->aeq.q_size * c2dev->aeq.msg_size,
598 q2_pages, dma_unmap_addr(&c2dev->aeq, mapping));
599 bail2:
600 dma_free_coherent(&c2dev->pcidev->dev,
601 c2dev->rep_vq.q_size * c2dev->rep_vq.msg_size,
602 q1_pages, dma_unmap_addr(&c2dev->rep_vq, mapping));
603 bail1:
604 c2_free_mqsp_pool(c2dev, c2dev->kern_mqsp_pool);
605 bail0:
606 vfree(c2dev->qptr_array);
607
608 return err;
609}
610
611
612
613
614void c2_rnic_term(struct c2_dev *c2dev)
615{
616
617
618 c2_rnic_close(c2dev);
619
620
621 c2_adapter_term(c2dev);
622
623
624 writel(1, c2dev->regs + C2_IDIS);
625
626
627 c2_cleanup_qp_table(c2dev);
628
629
630 c2_cleanup_pd_table(c2dev);
631
632
633 vq_term(c2dev);
634
635
636 dma_free_coherent(&c2dev->pcidev->dev,
637 c2dev->aeq.q_size * c2dev->aeq.msg_size,
638 c2dev->aeq.msg_pool.host,
639 dma_unmap_addr(&c2dev->aeq, mapping));
640
641
642 dma_free_coherent(&c2dev->pcidev->dev,
643 c2dev->rep_vq.q_size * c2dev->rep_vq.msg_size,
644 c2dev->rep_vq.msg_pool.host,
645 dma_unmap_addr(&c2dev->rep_vq, mapping));
646
647
648 c2_free_mqsp_pool(c2dev, c2dev->kern_mqsp_pool);
649
650
651 vfree(c2dev->qptr_array);
652
653 return;
654}
655
