1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35#ifndef I40IW_IW_H
36#define I40IW_IW_H
37#include <linux/netdevice.h>
38#include <linux/inetdevice.h>
39#include <linux/spinlock.h>
40#include <linux/kernel.h>
41#include <linux/delay.h>
42#include <linux/pci.h>
43#include <linux/dma-mapping.h>
44#include <linux/workqueue.h>
45#include <linux/slab.h>
46#include <linux/io.h>
47#include <linux/crc32c.h>
48#include <rdma/ib_smi.h>
49#include <rdma/ib_verbs.h>
50#include <rdma/ib_pack.h>
51#include <rdma/rdma_cm.h>
52#include <rdma/iw_cm.h>
53#include <rdma/iw_portmap.h>
54#include <rdma/rdma_netlink.h>
55#include <crypto/hash.h>
56
57#include "i40iw_status.h"
58#include "i40iw_osdep.h"
59#include "i40iw_d.h"
60#include "i40iw_hmc.h"
61
62#include <i40e_client.h>
63#include "i40iw_type.h"
64#include "i40iw_p.h"
65#include "i40iw_ucontext.h"
66#include "i40iw_pble.h"
67#include "i40iw_verbs.h"
68#include "i40iw_cm.h"
69#include "i40iw_user.h"
70#include "i40iw_puda.h"
71
72#define I40IW_FW_VERSION 2
73#define I40IW_HW_VERSION 2
74
75#define I40IW_ARP_ADD 1
76#define I40IW_ARP_DELETE 2
77#define I40IW_ARP_RESOLVE 3
78
79#define I40IW_MACIP_ADD 1
80#define I40IW_MACIP_DELETE 2
81
82#define IW_CCQ_SIZE (I40IW_CQP_SW_SQSIZE_2048 + 1)
83#define IW_CEQ_SIZE 2048
84#define IW_AEQ_SIZE 2048
85
86#define RX_BUF_SIZE (1536 + 8)
87#define IW_REG0_SIZE (4 * 1024)
88#define IW_TX_TIMEOUT (6 * HZ)
89#define IW_FIRST_QPN 1
90#define IW_SW_CONTEXT_ALIGN 1024
91
92#define MAX_DPC_ITERATIONS 128
93
94#define I40IW_EVENT_TIMEOUT 100000
95#define I40IW_VCHNL_EVENT_TIMEOUT 100000
96
97#define I40IW_NO_VLAN 0xffff
98#define I40IW_NO_QSET 0xffff
99
100
101#define IW_ADD_MCAST false
102#define IW_DEL_MCAST true
103
104#define I40IW_DRV_OPT_ENABLE_MPA_VER_0 0x00000001
105#define I40IW_DRV_OPT_DISABLE_MPA_CRC 0x00000002
106#define I40IW_DRV_OPT_DISABLE_FIRST_WRITE 0x00000004
107#define I40IW_DRV_OPT_DISABLE_INTF 0x00000008
108#define I40IW_DRV_OPT_ENABLE_MSI 0x00000010
109#define I40IW_DRV_OPT_DUAL_LOGICAL_PORT 0x00000020
110#define I40IW_DRV_OPT_NO_INLINE_DATA 0x00000080
111#define I40IW_DRV_OPT_DISABLE_INT_MOD 0x00000100
112#define I40IW_DRV_OPT_DISABLE_VIRT_WQ 0x00000200
113#define I40IW_DRV_OPT_ENABLE_PAU 0x00000400
114#define I40IW_DRV_OPT_MCAST_LOGPORT_MAP 0x00000800
115
116#define IW_HMC_OBJ_TYPE_NUM ARRAY_SIZE(iw_hmc_obj_types)
117#define IW_CFG_FPM_QP_COUNT 32768
118
119#define I40IW_MTU_TO_MSS 40
120#define I40IW_DEFAULT_MSS 1460
121
122struct i40iw_cqp_compl_info {
123 u32 op_ret_val;
124 u16 maj_err_code;
125 u16 min_err_code;
126 bool error;
127 u8 op_code;
128};
129
130#define i40iw_pr_err(fmt, args ...) pr_err("%s: "fmt, __func__, ## args)
131
132#define i40iw_pr_info(fmt, args ...) pr_info("%s: " fmt, __func__, ## args)
133
134#define i40iw_pr_warn(fmt, args ...) pr_warn("%s: " fmt, __func__, ## args)
135
136struct i40iw_cqp_request {
137 struct cqp_commands_info info;
138 wait_queue_head_t waitq;
139 struct list_head list;
140 atomic_t refcount;
141 void (*callback_fcn)(struct i40iw_cqp_request*, u32);
142 void *param;
143 struct i40iw_cqp_compl_info compl_info;
144 bool waiting;
145 bool request_done;
146 bool dynamic;
147};
148
149struct i40iw_cqp {
150 struct i40iw_sc_cqp sc_cqp;
151 spinlock_t req_lock;
152 wait_queue_head_t waitq;
153 struct i40iw_dma_mem sq;
154 struct i40iw_dma_mem host_ctx;
155 u64 *scratch_array;
156 struct i40iw_cqp_request *cqp_requests;
157 struct list_head cqp_avail_reqs;
158 struct list_head cqp_pending_reqs;
159};
160
161struct i40iw_device;
162
163struct i40iw_ccq {
164 struct i40iw_sc_cq sc_cq;
165 spinlock_t lock;
166 wait_queue_head_t waitq;
167 struct i40iw_dma_mem mem_cq;
168 struct i40iw_dma_mem shadow_area;
169};
170
171struct i40iw_ceq {
172 struct i40iw_sc_ceq sc_ceq;
173 struct i40iw_dma_mem mem;
174 u32 irq;
175 u32 msix_idx;
176 struct i40iw_device *iwdev;
177 struct tasklet_struct dpc_tasklet;
178};
179
180struct i40iw_aeq {
181 struct i40iw_sc_aeq sc_aeq;
182 struct i40iw_dma_mem mem;
183};
184
185struct i40iw_arp_entry {
186 u32 ip_addr[4];
187 u8 mac_addr[ETH_ALEN];
188};
189
190enum init_completion_state {
191 INVALID_STATE = 0,
192 INITIAL_STATE,
193 CQP_CREATED,
194 HMC_OBJS_CREATED,
195 PBLE_CHUNK_MEM,
196 CCQ_CREATED,
197 AEQ_CREATED,
198 CEQ_CREATED,
199 ILQ_CREATED,
200 IEQ_CREATED,
201 INET_NOTIFIER,
202 IP_ADDR_REGISTERED,
203 RDMA_DEV_REGISTERED
204};
205
206struct i40iw_msix_vector {
207 u32 idx;
208 u32 irq;
209 u32 cpu_affinity;
210 u32 ceq_id;
211};
212
213#define I40IW_MSIX_TABLE_SIZE 65
214
215struct virtchnl_work {
216 struct work_struct work;
217 union {
218 struct i40iw_cqp_request *cqp_request;
219 struct i40iw_virtchnl_work_info work_info;
220 };
221};
222
223struct i40e_qvlist_info;
224
225struct i40iw_device {
226 struct i40iw_ib_device *iwibdev;
227 struct net_device *netdev;
228 wait_queue_head_t vchnl_waitq;
229 struct i40iw_sc_dev sc_dev;
230 struct i40iw_handler *hdl;
231 struct i40e_info *ldev;
232 struct i40e_client *client;
233 struct i40iw_hw hw;
234 struct i40iw_cm_core cm_core;
235 unsigned long *mem_resources;
236 unsigned long *allocated_qps;
237 unsigned long *allocated_cqs;
238 unsigned long *allocated_mrs;
239 unsigned long *allocated_pds;
240 unsigned long *allocated_arps;
241 struct i40iw_qp **qp_table;
242 bool msix_shared;
243 u32 msix_count;
244 struct i40iw_msix_vector *iw_msixtbl;
245 struct i40e_qvlist_info *iw_qvlist;
246
247 struct i40iw_hmc_pble_rsrc *pble_rsrc;
248 struct i40iw_arp_entry *arp_table;
249 struct i40iw_cqp cqp;
250 struct i40iw_ccq ccq;
251 u32 ceqs_count;
252 struct i40iw_ceq *ceqlist;
253 struct i40iw_aeq aeq;
254 u32 arp_table_size;
255 u32 next_arp_index;
256 spinlock_t resource_lock;
257 u32 vendor_id;
258 u32 vendor_part_id;
259 u32 of_device_registered;
260
261 u32 device_cap_flags;
262 unsigned long db_start;
263 u8 resource_profile;
264 u8 max_rdma_vfs;
265 u8 max_enabled_vfs;
266 u8 max_sge;
267 u8 iw_status;
268 u8 send_term_ok;
269 bool push_mode;
270
271
272 struct mutex pbl_mutex;
273 struct tasklet_struct dpc_tasklet;
274 struct workqueue_struct *virtchnl_wq;
275 struct virtchnl_work virtchnl_w[I40IW_MAX_PE_ENABLED_VF_COUNT];
276 struct i40iw_dma_mem obj_mem;
277 struct i40iw_dma_mem obj_next;
278 u8 *hmc_info_mem;
279 u32 sd_type;
280 struct workqueue_struct *param_wq;
281 atomic_t params_busy;
282 u32 mss;
283 enum init_completion_state init_state;
284 u16 mac_ip_table_idx;
285 atomic_t vchnl_msgs;
286 u32 max_mr;
287 u32 max_qp;
288 u32 max_cq;
289 u32 max_pd;
290 u32 next_qp;
291 u32 next_cq;
292 u32 next_pd;
293 u32 max_mr_size;
294 u32 max_qp_wr;
295 u32 max_cqe;
296 u32 mr_stagmask;
297 u32 mpa_version;
298 bool dcb;
299};
300
301struct i40iw_ib_device {
302 struct ib_device ibdev;
303 struct i40iw_device *iwdev;
304};
305
306struct i40iw_handler {
307 struct list_head list;
308 struct i40e_client *client;
309 struct i40iw_device device;
310 struct i40e_info ldev;
311};
312
313
314
315
316
317static inline struct i40iw_device *to_iwdev(struct ib_device *ibdev)
318{
319 return container_of(ibdev, struct i40iw_ib_device, ibdev)->iwdev;
320}
321
322
323
324
325
326static inline struct i40iw_ucontext *to_ucontext(struct ib_ucontext *ibucontext)
327{
328 return container_of(ibucontext, struct i40iw_ucontext, ibucontext);
329}
330
331
332
333
334
335static inline struct i40iw_pd *to_iwpd(struct ib_pd *ibpd)
336{
337 return container_of(ibpd, struct i40iw_pd, ibpd);
338}
339
340
341
342
343
344static inline struct i40iw_mr *to_iwmr(struct ib_mr *ibmr)
345{
346 return container_of(ibmr, struct i40iw_mr, ibmr);
347}
348
349
350
351
352
353static inline struct i40iw_mr *to_iwmr_from_ibfmr(struct ib_fmr *ibfmr)
354{
355 return container_of(ibfmr, struct i40iw_mr, ibfmr);
356}
357
358
359
360
361
362static inline struct i40iw_mr *to_iwmw(struct ib_mw *ibmw)
363{
364 return container_of(ibmw, struct i40iw_mr, ibmw);
365}
366
367
368
369
370
371static inline struct i40iw_cq *to_iwcq(struct ib_cq *ibcq)
372{
373 return container_of(ibcq, struct i40iw_cq, ibcq);
374}
375
376
377
378
379
380static inline struct i40iw_qp *to_iwqp(struct ib_qp *ibqp)
381{
382 return container_of(ibqp, struct i40iw_qp, ibqp);
383}
384
385
386void i40iw_add_ref(struct ib_qp *);
387void i40iw_rem_ref(struct ib_qp *);
388struct ib_qp *i40iw_get_qp(struct ib_device *, int);
389
390void i40iw_flush_wqes(struct i40iw_device *iwdev,
391 struct i40iw_qp *qp);
392
393void i40iw_manage_arp_cache(struct i40iw_device *iwdev,
394 unsigned char *mac_addr,
395 __be32 *ip_addr,
396 bool ipv4,
397 u32 action);
398
399int i40iw_manage_apbvt(struct i40iw_device *iwdev,
400 u16 accel_local_port,
401 bool add_port);
402
403struct i40iw_cqp_request *i40iw_get_cqp_request(struct i40iw_cqp *cqp, bool wait);
404void i40iw_free_cqp_request(struct i40iw_cqp *cqp, struct i40iw_cqp_request *cqp_request);
405void i40iw_put_cqp_request(struct i40iw_cqp *cqp, struct i40iw_cqp_request *cqp_request);
406
407
408
409
410
411
412
413
414
415static inline int i40iw_alloc_resource(struct i40iw_device *iwdev,
416 unsigned long *resource_array,
417 u32 max_resources,
418 u32 *req_resource_num,
419 u32 *next)
420{
421 u32 resource_num;
422 unsigned long flags;
423
424 spin_lock_irqsave(&iwdev->resource_lock, flags);
425 resource_num = find_next_zero_bit(resource_array, max_resources, *next);
426 if (resource_num >= max_resources) {
427 resource_num = find_first_zero_bit(resource_array, max_resources);
428 if (resource_num >= max_resources) {
429 spin_unlock_irqrestore(&iwdev->resource_lock, flags);
430 return -EOVERFLOW;
431 }
432 }
433 set_bit(resource_num, resource_array);
434 *next = resource_num + 1;
435 if (*next == max_resources)
436 *next = 0;
437 spin_unlock_irqrestore(&iwdev->resource_lock, flags);
438 *req_resource_num = resource_num;
439
440 return 0;
441}
442
443
444
445
446
447
448
449
450static inline bool i40iw_is_resource_allocated(struct i40iw_device *iwdev,
451 unsigned long *resource_array,
452 u32 resource_num)
453{
454 bool bit_is_set;
455 unsigned long flags;
456
457 spin_lock_irqsave(&iwdev->resource_lock, flags);
458
459 bit_is_set = test_bit(resource_num, resource_array);
460 spin_unlock_irqrestore(&iwdev->resource_lock, flags);
461
462 return bit_is_set;
463}
464
465
466
467
468
469
470
471static inline void i40iw_free_resource(struct i40iw_device *iwdev,
472 unsigned long *resource_array,
473 u32 resource_num)
474{
475 unsigned long flags;
476
477 spin_lock_irqsave(&iwdev->resource_lock, flags);
478 clear_bit(resource_num, resource_array);
479 spin_unlock_irqrestore(&iwdev->resource_lock, flags);
480}
481
482
483
484
485
486static inline struct i40iw_handler *to_iwhdl(struct i40iw_device *iw_dev)
487{
488 return container_of(iw_dev, struct i40iw_handler, device);
489}
490
491struct i40iw_handler *i40iw_find_netdev(struct net_device *netdev);
492
493
494
495
496u32 i40iw_initialize_hw_resources(struct i40iw_device *iwdev);
497
498int i40iw_register_rdma_device(struct i40iw_device *iwdev);
499void i40iw_port_ibevent(struct i40iw_device *iwdev);
500int i40iw_cm_disconn(struct i40iw_qp *);
501void i40iw_cm_disconn_worker(void *);
502int mini_cm_recv_pkt(struct i40iw_cm_core *, struct i40iw_device *,
503 struct sk_buff *);
504
505enum i40iw_status_code i40iw_handle_cqp_op(struct i40iw_device *iwdev,
506 struct i40iw_cqp_request *cqp_request);
507enum i40iw_status_code i40iw_add_mac_addr(struct i40iw_device *iwdev,
508 u8 *mac_addr, u8 *mac_index);
509int i40iw_modify_qp(struct ib_qp *, struct ib_qp_attr *, int, struct ib_udata *);
510
511void i40iw_rem_pdusecount(struct i40iw_pd *iwpd, struct i40iw_device *iwdev);
512void i40iw_add_pdusecount(struct i40iw_pd *iwpd);
513void i40iw_hw_modify_qp(struct i40iw_device *iwdev, struct i40iw_qp *iwqp,
514 struct i40iw_modify_qp_info *info, bool wait);
515
516enum i40iw_status_code i40iw_manage_qhash(struct i40iw_device *iwdev,
517 struct i40iw_cm_info *cminfo,
518 enum i40iw_quad_entry_type etype,
519 enum i40iw_quad_hash_manage_type mtype,
520 void *cmnode,
521 bool wait);
522void i40iw_receive_ilq(struct i40iw_sc_dev *dev, struct i40iw_puda_buf *rbuf);
523void i40iw_free_sqbuf(struct i40iw_sc_dev *dev, void *bufp);
524void i40iw_free_qp_resources(struct i40iw_device *iwdev,
525 struct i40iw_qp *iwqp,
526 u32 qp_num);
527enum i40iw_status_code i40iw_obj_aligned_mem(struct i40iw_device *iwdev,
528 struct i40iw_dma_mem *memptr,
529 u32 size, u32 mask);
530
531void i40iw_request_reset(struct i40iw_device *iwdev);
532void i40iw_destroy_rdma_device(struct i40iw_ib_device *iwibdev);
533void i40iw_setup_cm_core(struct i40iw_device *iwdev);
534void i40iw_cleanup_cm_core(struct i40iw_cm_core *cm_core);
535void i40iw_process_ceq(struct i40iw_device *, struct i40iw_ceq *iwceq);
536void i40iw_process_aeq(struct i40iw_device *);
537void i40iw_next_iw_state(struct i40iw_qp *iwqp,
538 u8 state, u8 del_hash,
539 u8 term, u8 term_len);
540int i40iw_send_syn(struct i40iw_cm_node *cm_node, u32 sendack);
541struct i40iw_cm_node *i40iw_find_node(struct i40iw_cm_core *cm_core,
542 u16 rem_port,
543 u32 *rem_addr,
544 u16 loc_port,
545 u32 *loc_addr,
546 bool add_refcnt);
547
548enum i40iw_status_code i40iw_hw_flush_wqes(struct i40iw_device *iwdev,
549 struct i40iw_sc_qp *qp,
550 struct i40iw_qp_flush_info *info,
551 bool wait);
552
553void i40iw_copy_ip_ntohl(u32 *dst, u32 *src);
554struct ib_mr *i40iw_reg_phys_mr(struct ib_pd *ib_pd,
555 u64 addr,
556 u64 size,
557 int acc,
558 u64 *iova_start);
559
560int i40iw_inetaddr_event(struct notifier_block *notifier,
561 unsigned long event,
562 void *ptr);
563int i40iw_inet6addr_event(struct notifier_block *notifier,
564 unsigned long event,
565 void *ptr);
566int i40iw_net_event(struct notifier_block *notifier,
567 unsigned long event,
568 void *ptr);
569
570#endif
571
