1#ifndef DEF_RDMA_VT_H
2#define DEF_RDMA_VT_H
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56#include <linux/spinlock.h>
57#include <linux/list.h>
58#include <linux/hash.h>
59#include <rdma/ib_verbs.h>
60#include <rdma/rdmavt_mr.h>
61#include <rdma/rdmavt_qp.h>
62
63#define RVT_MAX_PKEY_VALUES 16
64
65struct rvt_ibport {
66 struct rvt_qp __rcu *qp[2];
67 struct ib_mad_agent *send_agent;
68 struct rb_root mcast_tree;
69 spinlock_t lock;
70
71
72 unsigned long mkey_lease_timeout;
73 unsigned long trap_timeout;
74 __be64 gid_prefix;
75 __be64 mkey;
76 u64 tid;
77 u32 port_cap_flags;
78 u32 pma_sample_start;
79 u32 pma_sample_interval;
80 __be16 pma_counter_select[5];
81 u16 pma_tag;
82 u16 mkey_lease_period;
83 u16 sm_lid;
84 u8 sm_sl;
85 u8 mkeyprot;
86 u8 subnet_timeout;
87 u8 vl_high_limit;
88
89
90
91
92
93
94 u64 n_rc_resends;
95 u64 n_seq_naks;
96 u64 n_rdma_seq;
97 u64 n_rnr_naks;
98 u64 n_other_naks;
99 u64 n_loop_pkts;
100 u64 n_pkt_drops;
101 u64 n_vl15_dropped;
102 u64 n_rc_timeouts;
103 u64 n_dmawait;
104 u64 n_unaligned;
105 u64 n_rc_dupreq;
106 u64 n_rc_seqnak;
107 u16 pkey_violations;
108 u16 qkey_violations;
109 u16 mkey_violations;
110
111
112 u64 z_rc_acks;
113 u64 z_rc_qacks;
114 u64 z_rc_delayed_comp;
115 u64 __percpu *rc_acks;
116 u64 __percpu *rc_qacks;
117 u64 __percpu *rc_delayed_comp;
118
119 void *priv;
120
121
122
123
124
125
126
127 u16 *pkey_table;
128
129 struct rvt_ah *sm_ah;
130};
131
132#define RVT_CQN_MAX 16
133
134
135
136
137struct rvt_driver_params {
138 struct ib_device_attr props;
139
140
141
142
143
144 unsigned int lkey_table_size;
145 unsigned int qp_table_size;
146 int qpn_start;
147 int qpn_inc;
148 int qpn_res_start;
149 int qpn_res_end;
150 int nports;
151 int npkeys;
152 char cq_name[RVT_CQN_MAX];
153 int node;
154 int psn_mask;
155 int psn_shift;
156 int psn_modify_mask;
157 u32 core_cap_flags;
158 u32 max_mad_size;
159 u8 qos_shift;
160 u8 max_rdma_atomic;
161 u8 reserved_operations;
162};
163
164
165struct rvt_pd {
166 struct ib_pd ibpd;
167 int user;
168};
169
170
171struct rvt_ah {
172 struct ib_ah ibah;
173 struct ib_ah_attr attr;
174 atomic_t refcount;
175 u8 vl;
176 u8 log_pmtu;
177};
178
179struct rvt_dev_info;
180struct rvt_swqe;
181struct rvt_driver_provided {
182
183
184
185
186
187
188
189 int (*port_callback)(struct ib_device *, u8, struct kobject *);
190
191
192
193
194
195
196 const char * (*get_card_name)(struct rvt_dev_info *rdi);
197
198
199
200
201
202
203 struct pci_dev * (*get_pci_dev)(struct rvt_dev_info *rdi);
204
205
206
207
208
209
210
211 void * (*qp_priv_alloc)(struct rvt_dev_info *rdi, struct rvt_qp *qp,
212 gfp_t gfp);
213
214
215
216
217 void (*qp_priv_free)(struct rvt_dev_info *rdi, struct rvt_qp *qp);
218
219
220
221
222
223 void (*notify_qp_reset)(struct rvt_qp *qp);
224
225
226
227
228
229
230
231
232 void (*schedule_send)(struct rvt_qp *qp);
233 void (*schedule_send_no_lock)(struct rvt_qp *qp);
234
235
236
237
238
239 void (*do_send)(struct rvt_qp *qp);
240
241
242
243
244 int (*get_pmtu_from_attr)(struct rvt_dev_info *rdi, struct rvt_qp *qp,
245 struct ib_qp_attr *attr);
246
247
248
249
250
251 void (*flush_qp_waiters)(struct rvt_qp *qp);
252
253
254
255
256
257 void (*stop_send_queue)(struct rvt_qp *qp);
258
259
260
261
262 void (*quiesce_qp)(struct rvt_qp *qp);
263
264
265
266
267 void (*notify_error_qp)(struct rvt_qp *qp);
268
269
270
271
272 u32 (*mtu_from_qp)(struct rvt_dev_info *rdi, struct rvt_qp *qp,
273 u32 pmtu);
274
275
276
277 int (*mtu_to_path_mtu)(u32 mtu);
278
279
280
281
282 int (*get_guid_be)(struct rvt_dev_info *rdi, struct rvt_ibport *rvp,
283 int guid_index, __be64 *guid);
284
285
286
287
288 int (*query_port_state)(struct rvt_dev_info *rdi, u8 port_num,
289 struct ib_port_attr *props);
290
291
292
293
294 int (*shut_down_port)(struct rvt_dev_info *rdi, u8 port_num);
295
296
297 void (*cap_mask_chg)(struct rvt_dev_info *rdi, u8 port_num);
298
299
300
301
302
303
304
305
306 unsigned (*free_all_qps)(struct rvt_dev_info *rdi);
307
308
309 int (*check_ah)(struct ib_device *, struct ib_ah_attr *);
310
311
312 void (*notify_new_ah)(struct ib_device *, struct ib_ah_attr *,
313 struct rvt_ah *);
314
315
316 int (*alloc_qpn)(struct rvt_dev_info *rdi, struct rvt_qpn_table *qpt,
317 enum ib_qp_type type, u8 port_num, gfp_t gfp);
318
319
320 int (*check_modify_qp)(struct rvt_qp *qp, struct ib_qp_attr *attr,
321 int attr_mask, struct ib_udata *udata);
322
323
324 void (*modify_qp)(struct rvt_qp *qp, struct ib_qp_attr *attr,
325 int attr_mask, struct ib_udata *udata);
326
327
328 int (*check_send_wqe)(struct rvt_qp *qp, struct rvt_swqe *wqe);
329
330
331 void (*notify_create_mad_agent)(struct rvt_dev_info *rdi, int port_idx);
332
333
334 void (*notify_free_mad_agent)(struct rvt_dev_info *rdi, int port_idx);
335
336};
337
338struct rvt_dev_info {
339 struct ib_device ibdev;
340
341
342
343
344
345
346
347
348
349
350
351
352
353 struct rvt_driver_params dparms;
354
355
356 const struct rvt_operation_params *post_parms;
357
358 struct rvt_mregion __rcu *dma_mr;
359 struct rvt_lkey_table lkey_table;
360
361
362 struct rvt_driver_provided driver_f;
363
364
365 int n_pds_allocated;
366 spinlock_t n_pds_lock;
367
368 int n_ahs_allocated;
369 spinlock_t n_ahs_lock;
370
371 u32 n_srqs_allocated;
372 spinlock_t n_srqs_lock;
373
374 int flags;
375 struct rvt_ibport **ports;
376
377
378 struct rvt_qp_ibdev *qp_dev;
379 u32 n_qps_allocated;
380 u32 n_rc_qps;
381 u32 busy_jiffies;
382 spinlock_t n_qps_lock;
383
384
385 struct list_head pending_mmaps;
386 spinlock_t mmap_offset_lock;
387 u32 mmap_offset;
388 spinlock_t pending_lock;
389
390
391 struct kthread_worker *worker;
392 u32 n_cqs_allocated;
393 spinlock_t n_cqs_lock;
394
395
396 u32 n_mcast_grps_allocated;
397 spinlock_t n_mcast_grps_lock;
398
399};
400
401static inline struct rvt_pd *ibpd_to_rvtpd(struct ib_pd *ibpd)
402{
403 return container_of(ibpd, struct rvt_pd, ibpd);
404}
405
406static inline struct rvt_ah *ibah_to_rvtah(struct ib_ah *ibah)
407{
408 return container_of(ibah, struct rvt_ah, ibah);
409}
410
411static inline struct rvt_dev_info *ib_to_rvt(struct ib_device *ibdev)
412{
413 return container_of(ibdev, struct rvt_dev_info, ibdev);
414}
415
416static inline struct rvt_srq *ibsrq_to_rvtsrq(struct ib_srq *ibsrq)
417{
418 return container_of(ibsrq, struct rvt_srq, ibsrq);
419}
420
421static inline struct rvt_qp *ibqp_to_rvtqp(struct ib_qp *ibqp)
422{
423 return container_of(ibqp, struct rvt_qp, ibqp);
424}
425
426static inline unsigned rvt_get_npkeys(struct rvt_dev_info *rdi)
427{
428
429
430
431 return rdi->dparms.npkeys;
432}
433
434
435
436
437
438static inline unsigned int rvt_max_atomic(struct rvt_dev_info *rdi)
439{
440 return rdi->dparms.max_rdma_atomic + 1;
441}
442
443
444
445
446static inline u16 rvt_get_pkey(struct rvt_dev_info *rdi,
447 int port_index,
448 unsigned index)
449{
450 if (index >= rvt_get_npkeys(rdi))
451 return 0;
452 else
453 return rdi->ports[port_index]->pkey_table[index];
454}
455
456
457
458
459
460
461
462
463
464
465static inline struct rvt_qp *rvt_lookup_qpn(struct rvt_dev_info *rdi,
466 struct rvt_ibport *rvp,
467 u32 qpn) __must_hold(RCU)
468{
469 struct rvt_qp *qp = NULL;
470
471 if (unlikely(qpn <= 1)) {
472 qp = rcu_dereference(rvp->qp[qpn]);
473 } else {
474 u32 n = hash_32(qpn, rdi->qp_dev->qp_table_bits);
475
476 for (qp = rcu_dereference(rdi->qp_dev->qp_table[n]); qp;
477 qp = rcu_dereference(qp->next))
478 if (qp->ibqp.qp_num == qpn)
479 break;
480 }
481 return qp;
482}
483
484struct rvt_dev_info *rvt_alloc_device(size_t size, int nports);
485void rvt_dealloc_device(struct rvt_dev_info *rdi);
486int rvt_register_device(struct rvt_dev_info *rvd);
487void rvt_unregister_device(struct rvt_dev_info *rvd);
488int rvt_check_ah(struct ib_device *ibdev, struct ib_ah_attr *ah_attr);
489int rvt_init_port(struct rvt_dev_info *rdi, struct rvt_ibport *port,
490 int port_index, u16 *pkey_table);
491int rvt_fast_reg_mr(struct rvt_qp *qp, struct ib_mr *ibmr, u32 key,
492 int access);
493int rvt_invalidate_rkey(struct rvt_qp *qp, u32 rkey);
494int rvt_rkey_ok(struct rvt_qp *qp, struct rvt_sge *sge,
495 u32 len, u64 vaddr, u32 rkey, int acc);
496int rvt_lkey_ok(struct rvt_lkey_table *rkt, struct rvt_pd *pd,
497 struct rvt_sge *isge, struct ib_sge *sge, int acc);
498struct rvt_mcast *rvt_mcast_find(struct rvt_ibport *ibp, union ib_gid *mgid);
499
500#endif
501