1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35#ifndef I40IW_USER_H
36#define I40IW_USER_H
37
38enum i40iw_device_capabilities_const {
39 I40IW_WQE_SIZE = 4,
40 I40IW_CQP_WQE_SIZE = 8,
41 I40IW_CQE_SIZE = 4,
42 I40IW_EXTENDED_CQE_SIZE = 8,
43 I40IW_AEQE_SIZE = 2,
44 I40IW_CEQE_SIZE = 1,
45 I40IW_CQP_CTX_SIZE = 8,
46 I40IW_SHADOW_AREA_SIZE = 8,
47 I40IW_CEQ_MAX_COUNT = 256,
48 I40IW_QUERY_FPM_BUF_SIZE = 128,
49 I40IW_COMMIT_FPM_BUF_SIZE = 128,
50 I40IW_MIN_IW_QP_ID = 1,
51 I40IW_MAX_IW_QP_ID = 262143,
52 I40IW_MIN_CEQID = 0,
53 I40IW_MAX_CEQID = 256,
54 I40IW_MIN_CQID = 0,
55 I40IW_MAX_CQID = 131071,
56 I40IW_MIN_AEQ_ENTRIES = 1,
57 I40IW_MAX_AEQ_ENTRIES = 524287,
58 I40IW_MIN_CEQ_ENTRIES = 1,
59 I40IW_MAX_CEQ_ENTRIES = 131071,
60 I40IW_MIN_CQ_SIZE = 1,
61 I40IW_MAX_CQ_SIZE = 1048575,
62 I40IW_MAX_AEQ_ALLOCATE_COUNT = 255,
63 I40IW_DB_ID_ZERO = 0,
64 I40IW_MAX_WQ_FRAGMENT_COUNT = 6,
65 I40IW_MAX_SGE_RD = 1,
66 I40IW_MAX_OUTBOUND_MESSAGE_SIZE = 2147483647,
67 I40IW_MAX_INBOUND_MESSAGE_SIZE = 2147483647,
68 I40IW_MAX_PUSH_PAGE_COUNT = 4096,
69 I40IW_MAX_PE_ENABLED_VF_COUNT = 32,
70 I40IW_MAX_VF_FPM_ID = 47,
71 I40IW_MAX_VF_PER_PF = 127,
72 I40IW_MAX_SQ_PAYLOAD_SIZE = 2145386496,
73 I40IW_MAX_INLINE_DATA_SIZE = 112,
74 I40IW_MAX_PUSHMODE_INLINE_DATA_SIZE = 112,
75 I40IW_MAX_IRD_SIZE = 32,
76 I40IW_QPCTX_ENCD_MAXIRD = 3,
77 I40IW_MAX_WQ_ENTRIES = 2048,
78 I40IW_MAX_ORD_SIZE = 32,
79 I40IW_Q2_BUFFER_SIZE = (248 + 100),
80 I40IW_QP_CTX_SIZE = 248
81};
82
83#define i40iw_handle void *
84#define i40iw_adapter_handle i40iw_handle
85#define i40iw_qp_handle i40iw_handle
86#define i40iw_cq_handle i40iw_handle
87#define i40iw_srq_handle i40iw_handle
88#define i40iw_pd_id i40iw_handle
89#define i40iw_stag_handle i40iw_handle
90#define i40iw_stag_index u32
91#define i40iw_stag u32
92#define i40iw_stag_key u8
93
94#define i40iw_tagged_offset u64
95#define i40iw_access_privileges u32
96#define i40iw_physical_fragment u64
97#define i40iw_address_list u64 *
98
99#define I40IW_CREATE_STAG(index, key) (((index) << 8) + (key))
100
101#define I40IW_STAG_KEY_FROM_STAG(stag) ((stag) && 0x000000FF)
102
103#define I40IW_STAG_INDEX_FROM_STAG(stag) (((stag) && 0xFFFFFF00) >> 8)
104
105struct i40iw_qp_uk;
106struct i40iw_cq_uk;
107struct i40iw_srq_uk;
108struct i40iw_qp_uk_init_info;
109struct i40iw_cq_uk_init_info;
110struct i40iw_srq_uk_init_info;
111
112struct i40iw_sge {
113 i40iw_tagged_offset tag_off;
114 u32 len;
115 i40iw_stag stag;
116};
117
118#define i40iw_sgl struct i40iw_sge *
119
120struct i40iw_ring {
121 u32 head;
122 u32 tail;
123 u32 size;
124};
125
126struct i40iw_cqe {
127 u64 buf[I40IW_CQE_SIZE];
128};
129
130struct i40iw_extended_cqe {
131 u64 buf[I40IW_EXTENDED_CQE_SIZE];
132};
133
134struct i40iw_wqe {
135 u64 buf[I40IW_WQE_SIZE];
136};
137
138struct i40iw_qp_uk_ops;
139
140enum i40iw_addressing_type {
141 I40IW_ADDR_TYPE_ZERO_BASED = 0,
142 I40IW_ADDR_TYPE_VA_BASED = 1,
143};
144
145#define I40IW_ACCESS_FLAGS_LOCALREAD 0x01
146#define I40IW_ACCESS_FLAGS_LOCALWRITE 0x02
147#define I40IW_ACCESS_FLAGS_REMOTEREAD_ONLY 0x04
148#define I40IW_ACCESS_FLAGS_REMOTEREAD 0x05
149#define I40IW_ACCESS_FLAGS_REMOTEWRITE_ONLY 0x08
150#define I40IW_ACCESS_FLAGS_REMOTEWRITE 0x0a
151#define I40IW_ACCESS_FLAGS_BIND_WINDOW 0x10
152#define I40IW_ACCESS_FLAGS_ALL 0x1F
153
154#define I40IW_OP_TYPE_RDMA_WRITE 0
155#define I40IW_OP_TYPE_RDMA_READ 1
156#define I40IW_OP_TYPE_SEND 3
157#define I40IW_OP_TYPE_SEND_INV 4
158#define I40IW_OP_TYPE_SEND_SOL 5
159#define I40IW_OP_TYPE_SEND_SOL_INV 6
160#define I40IW_OP_TYPE_REC 7
161#define I40IW_OP_TYPE_BIND_MW 8
162#define I40IW_OP_TYPE_FAST_REG_NSMR 9
163#define I40IW_OP_TYPE_INV_STAG 10
164#define I40IW_OP_TYPE_RDMA_READ_INV_STAG 11
165#define I40IW_OP_TYPE_NOP 12
166
167enum i40iw_completion_status {
168 I40IW_COMPL_STATUS_SUCCESS = 0,
169 I40IW_COMPL_STATUS_FLUSHED,
170 I40IW_COMPL_STATUS_INVALID_WQE,
171 I40IW_COMPL_STATUS_QP_CATASTROPHIC,
172 I40IW_COMPL_STATUS_REMOTE_TERMINATION,
173 I40IW_COMPL_STATUS_INVALID_STAG,
174 I40IW_COMPL_STATUS_BASE_BOUND_VIOLATION,
175 I40IW_COMPL_STATUS_ACCESS_VIOLATION,
176 I40IW_COMPL_STATUS_INVALID_PD_ID,
177 I40IW_COMPL_STATUS_WRAP_ERROR,
178 I40IW_COMPL_STATUS_STAG_INVALID_PDID,
179 I40IW_COMPL_STATUS_RDMA_READ_ZERO_ORD,
180 I40IW_COMPL_STATUS_QP_NOT_PRIVLEDGED,
181 I40IW_COMPL_STATUS_STAG_NOT_INVALID,
182 I40IW_COMPL_STATUS_INVALID_PHYS_BUFFER_SIZE,
183 I40IW_COMPL_STATUS_INVALID_PHYS_BUFFER_ENTRY,
184 I40IW_COMPL_STATUS_INVALID_FBO,
185 I40IW_COMPL_STATUS_INVALID_LENGTH,
186 I40IW_COMPL_STATUS_INVALID_ACCESS,
187 I40IW_COMPL_STATUS_PHYS_BUFFER_LIST_TOO_LONG,
188 I40IW_COMPL_STATUS_INVALID_VIRT_ADDRESS,
189 I40IW_COMPL_STATUS_INVALID_REGION,
190 I40IW_COMPL_STATUS_INVALID_WINDOW,
191 I40IW_COMPL_STATUS_INVALID_TOTAL_LENGTH
192};
193
194enum i40iw_completion_notify {
195 IW_CQ_COMPL_EVENT = 0,
196 IW_CQ_COMPL_SOLICITED = 1
197};
198
199struct i40iw_post_send {
200 i40iw_sgl sg_list;
201 u8 num_sges;
202};
203
204struct i40iw_post_inline_send {
205 void *data;
206 u32 len;
207};
208
209struct i40iw_post_send_w_inv {
210 i40iw_sgl sg_list;
211 u32 num_sges;
212 i40iw_stag remote_stag_to_inv;
213};
214
215struct i40iw_post_inline_send_w_inv {
216 void *data;
217 u32 len;
218 i40iw_stag remote_stag_to_inv;
219};
220
221struct i40iw_rdma_write {
222 i40iw_sgl lo_sg_list;
223 u8 num_lo_sges;
224 struct i40iw_sge rem_addr;
225};
226
227struct i40iw_inline_rdma_write {
228 void *data;
229 u32 len;
230 struct i40iw_sge rem_addr;
231};
232
233struct i40iw_rdma_read {
234 struct i40iw_sge lo_addr;
235 struct i40iw_sge rem_addr;
236};
237
238struct i40iw_bind_window {
239 i40iw_stag mr_stag;
240 u64 bind_length;
241 void *va;
242 enum i40iw_addressing_type addressing_type;
243 bool enable_reads;
244 bool enable_writes;
245 i40iw_stag mw_stag;
246};
247
248struct i40iw_inv_local_stag {
249 i40iw_stag target_stag;
250};
251
252struct i40iw_post_sq_info {
253 u64 wr_id;
254 u8 op_type;
255 bool signaled;
256 bool read_fence;
257 bool local_fence;
258 bool inline_data;
259 bool defer_flag;
260 union {
261 struct i40iw_post_send send;
262 struct i40iw_post_send send_w_sol;
263 struct i40iw_post_send_w_inv send_w_inv;
264 struct i40iw_post_send_w_inv send_w_sol_inv;
265 struct i40iw_rdma_write rdma_write;
266 struct i40iw_rdma_read rdma_read;
267 struct i40iw_rdma_read rdma_read_inv;
268 struct i40iw_bind_window bind_window;
269 struct i40iw_inv_local_stag inv_local_stag;
270 struct i40iw_inline_rdma_write inline_rdma_write;
271 struct i40iw_post_inline_send inline_send;
272 struct i40iw_post_inline_send inline_send_w_sol;
273 struct i40iw_post_inline_send_w_inv inline_send_w_inv;
274 struct i40iw_post_inline_send_w_inv inline_send_w_sol_inv;
275 } op;
276};
277
278struct i40iw_post_rq_info {
279 u64 wr_id;
280 i40iw_sgl sg_list;
281 u32 num_sges;
282};
283
284struct i40iw_cq_poll_info {
285 u64 wr_id;
286 i40iw_qp_handle qp_handle;
287 u32 bytes_xfered;
288 u32 tcp_seq_num;
289 u32 qp_id;
290 i40iw_stag inv_stag;
291 enum i40iw_completion_status comp_status;
292 u16 major_err;
293 u16 minor_err;
294 u8 op_type;
295 bool stag_invalid_set;
296 bool push_dropped;
297 bool error;
298 bool is_srq;
299 bool solicited_event;
300};
301
302struct i40iw_qp_uk_ops {
303 void (*iw_qp_post_wr)(struct i40iw_qp_uk *);
304 void (*iw_qp_ring_push_db)(struct i40iw_qp_uk *, u32);
305 enum i40iw_status_code (*iw_rdma_write)(struct i40iw_qp_uk *,
306 struct i40iw_post_sq_info *, bool);
307 enum i40iw_status_code (*iw_rdma_read)(struct i40iw_qp_uk *,
308 struct i40iw_post_sq_info *, bool, bool);
309 enum i40iw_status_code (*iw_send)(struct i40iw_qp_uk *,
310 struct i40iw_post_sq_info *, u32, bool);
311 enum i40iw_status_code (*iw_inline_rdma_write)(struct i40iw_qp_uk *,
312 struct i40iw_post_sq_info *, bool);
313 enum i40iw_status_code (*iw_inline_send)(struct i40iw_qp_uk *,
314 struct i40iw_post_sq_info *, u32, bool);
315 enum i40iw_status_code (*iw_stag_local_invalidate)(struct i40iw_qp_uk *,
316 struct i40iw_post_sq_info *, bool);
317 enum i40iw_status_code (*iw_mw_bind)(struct i40iw_qp_uk *,
318 struct i40iw_post_sq_info *, bool);
319 enum i40iw_status_code (*iw_post_receive)(struct i40iw_qp_uk *,
320 struct i40iw_post_rq_info *);
321 enum i40iw_status_code (*iw_post_nop)(struct i40iw_qp_uk *, u64, bool, bool);
322};
323
324struct i40iw_cq_ops {
325 void (*iw_cq_request_notification)(struct i40iw_cq_uk *,
326 enum i40iw_completion_notify);
327 enum i40iw_status_code (*iw_cq_poll_completion)(struct i40iw_cq_uk *,
328 struct i40iw_cq_poll_info *, bool);
329 enum i40iw_status_code (*iw_cq_post_entries)(struct i40iw_cq_uk *, u8 count);
330 void (*iw_cq_clean)(void *, struct i40iw_cq_uk *);
331};
332
333struct i40iw_dev_uk;
334
335struct i40iw_device_uk_ops {
336 enum i40iw_status_code (*iwarp_cq_uk_init)(struct i40iw_cq_uk *,
337 struct i40iw_cq_uk_init_info *);
338 enum i40iw_status_code (*iwarp_qp_uk_init)(struct i40iw_qp_uk *,
339 struct i40iw_qp_uk_init_info *);
340};
341
342struct i40iw_dev_uk {
343 struct i40iw_device_uk_ops ops_uk;
344};
345
346struct i40iw_sq_uk_wr_trk_info {
347 u64 wrid;
348 u64 wr_len;
349};
350
351struct i40iw_qp_quanta {
352 u64 elem[I40IW_WQE_SIZE];
353};
354
355struct i40iw_qp_uk {
356 struct i40iw_qp_quanta *sq_base;
357 struct i40iw_qp_quanta *rq_base;
358 u32 __iomem *wqe_alloc_reg;
359 struct i40iw_sq_uk_wr_trk_info *sq_wrtrk_array;
360 u64 *rq_wrid_array;
361 u64 *shadow_area;
362 u32 *push_db;
363 u64 *push_wqe;
364 struct i40iw_ring sq_ring;
365 struct i40iw_ring rq_ring;
366 struct i40iw_ring initial_ring;
367 u32 qp_id;
368 u32 sq_size;
369 u32 rq_size;
370 struct i40iw_qp_uk_ops ops;
371 bool use_srq;
372 u8 swqe_polarity;
373 u8 swqe_polarity_deferred;
374 u8 rwqe_polarity;
375 u8 rq_wqe_size;
376 u8 rq_wqe_size_multiplier;
377 u8 max_sq_frag_cnt;
378 u8 max_rq_frag_cnt;
379 bool deferred_flag;
380};
381
382struct i40iw_cq_uk {
383 struct i40iw_cqe *cq_base;
384 u32 __iomem *cqe_alloc_reg;
385 u64 *shadow_area;
386 u32 cq_id;
387 u32 cq_size;
388 struct i40iw_ring cq_ring;
389 u8 polarity;
390 bool avoid_mem_cflct;
391
392 struct i40iw_cq_ops ops;
393};
394
395struct i40iw_qp_uk_init_info {
396 struct i40iw_qp_quanta *sq;
397 struct i40iw_qp_quanta *rq;
398 u32 __iomem *wqe_alloc_reg;
399 u64 *shadow_area;
400 struct i40iw_sq_uk_wr_trk_info *sq_wrtrk_array;
401 u64 *rq_wrid_array;
402 u32 *push_db;
403 u64 *push_wqe;
404 u32 qp_id;
405 u32 sq_size;
406 u32 rq_size;
407 u8 max_sq_frag_cnt;
408 u8 max_rq_frag_cnt;
409
410};
411
412struct i40iw_cq_uk_init_info {
413 u32 __iomem *cqe_alloc_reg;
414 struct i40iw_cqe *cq_base;
415 u64 *shadow_area;
416 u32 cq_size;
417 u32 cq_id;
418 bool avoid_mem_cflct;
419};
420
421void i40iw_device_init_uk(struct i40iw_dev_uk *dev);
422
423void i40iw_qp_post_wr(struct i40iw_qp_uk *qp);
424u64 *i40iw_qp_get_next_send_wqe(struct i40iw_qp_uk *qp, u32 *wqe_idx,
425 u8 wqe_size);
426u64 *i40iw_qp_get_next_recv_wqe(struct i40iw_qp_uk *qp, u32 *wqe_idx);
427u64 *i40iw_qp_get_next_srq_wqe(struct i40iw_srq_uk *srq, u32 *wqe_idx);
428
429enum i40iw_status_code i40iw_cq_uk_init(struct i40iw_cq_uk *cq,
430 struct i40iw_cq_uk_init_info *info);
431enum i40iw_status_code i40iw_qp_uk_init(struct i40iw_qp_uk *qp,
432 struct i40iw_qp_uk_init_info *info);
433
434void i40iw_clean_cq(void *queue, struct i40iw_cq_uk *cq);
435enum i40iw_status_code i40iw_nop(struct i40iw_qp_uk *qp, u64 wr_id,
436 bool signaled, bool post_sq);
437enum i40iw_status_code i40iw_fragcnt_to_wqesize_sq(u8 frag_cnt, u8 *wqe_size);
438enum i40iw_status_code i40iw_fragcnt_to_wqesize_rq(u8 frag_cnt, u8 *wqe_size);
439enum i40iw_status_code i40iw_inline_data_size_to_wqesize(u32 data_size,
440 u8 *wqe_size);
441enum i40iw_status_code i40iw_get_wqe_shift(u32 wqdepth, u8 sge, u8 *shift);
442#endif
443
