1
2
3
4#ifndef _GDMA_H
5#define _GDMA_H
6
7#include <linux/dma-mapping.h>
8#include <linux/netdevice.h>
9
10#include "shm_channel.h"
11
12
13
14
15
16enum gdma_request_type {
17 GDMA_VERIFY_VF_DRIVER_VERSION = 1,
18 GDMA_QUERY_MAX_RESOURCES = 2,
19 GDMA_LIST_DEVICES = 3,
20 GDMA_REGISTER_DEVICE = 4,
21 GDMA_DEREGISTER_DEVICE = 5,
22 GDMA_GENERATE_TEST_EQE = 10,
23 GDMA_CREATE_QUEUE = 12,
24 GDMA_DISABLE_QUEUE = 13,
25 GDMA_CREATE_DMA_REGION = 25,
26 GDMA_DMA_REGION_ADD_PAGES = 26,
27 GDMA_DESTROY_DMA_REGION = 27,
28};
29
30enum gdma_queue_type {
31 GDMA_INVALID_QUEUE,
32 GDMA_SQ,
33 GDMA_RQ,
34 GDMA_CQ,
35 GDMA_EQ,
36};
37
38enum gdma_work_request_flags {
39 GDMA_WR_NONE = 0,
40 GDMA_WR_OOB_IN_SGL = BIT(0),
41 GDMA_WR_PAD_BY_SGE0 = BIT(1),
42};
43
44enum gdma_eqe_type {
45 GDMA_EQE_COMPLETION = 3,
46 GDMA_EQE_TEST_EVENT = 64,
47 GDMA_EQE_HWC_INIT_EQ_ID_DB = 129,
48 GDMA_EQE_HWC_INIT_DATA = 130,
49 GDMA_EQE_HWC_INIT_DONE = 131,
50};
51
52enum {
53 GDMA_DEVICE_NONE = 0,
54 GDMA_DEVICE_HWC = 1,
55 GDMA_DEVICE_MANA = 2,
56};
57
58struct gdma_resource {
59
60 spinlock_t lock;
61
62
63 u32 size;
64
65
66 unsigned long *map;
67};
68
69union gdma_doorbell_entry {
70 u64 as_uint64;
71
72 struct {
73 u64 id : 24;
74 u64 reserved : 8;
75 u64 tail_ptr : 31;
76 u64 arm : 1;
77 } cq;
78
79 struct {
80 u64 id : 24;
81 u64 wqe_cnt : 8;
82 u64 tail_ptr : 32;
83 } rq;
84
85 struct {
86 u64 id : 24;
87 u64 reserved : 8;
88 u64 tail_ptr : 32;
89 } sq;
90
91 struct {
92 u64 id : 16;
93 u64 reserved : 16;
94 u64 tail_ptr : 31;
95 u64 arm : 1;
96 } eq;
97};
98
99struct gdma_msg_hdr {
100 u32 hdr_type;
101 u32 msg_type;
102 u16 msg_version;
103 u16 hwc_msg_id;
104 u32 msg_size;
105};
106
107struct gdma_dev_id {
108 union {
109 struct {
110 u16 type;
111 u16 instance;
112 };
113
114 u32 as_uint32;
115 };
116};
117
118struct gdma_req_hdr {
119 struct gdma_msg_hdr req;
120 struct gdma_msg_hdr resp;
121 struct gdma_dev_id dev_id;
122 u32 activity_id;
123};
124
125struct gdma_resp_hdr {
126 struct gdma_msg_hdr response;
127 struct gdma_dev_id dev_id;
128 u32 activity_id;
129 u32 status;
130 u32 reserved;
131};
132
133struct gdma_general_req {
134 struct gdma_req_hdr hdr;
135};
136
137#define GDMA_MESSAGE_V1 1
138
139struct gdma_general_resp {
140 struct gdma_resp_hdr hdr;
141};
142
143#define GDMA_STANDARD_HEADER_TYPE 0
144
145static inline void mana_gd_init_req_hdr(struct gdma_req_hdr *hdr, u32 code,
146 u32 req_size, u32 resp_size)
147{
148 hdr->req.hdr_type = GDMA_STANDARD_HEADER_TYPE;
149 hdr->req.msg_type = code;
150 hdr->req.msg_version = GDMA_MESSAGE_V1;
151 hdr->req.msg_size = req_size;
152
153 hdr->resp.hdr_type = GDMA_STANDARD_HEADER_TYPE;
154 hdr->resp.msg_type = code;
155 hdr->resp.msg_version = GDMA_MESSAGE_V1;
156 hdr->resp.msg_size = resp_size;
157}
158
159
160struct gdma_sge {
161 u64 address;
162 u32 mem_key;
163 u32 size;
164};
165
166struct gdma_wqe_request {
167 struct gdma_sge *sgl;
168 u32 num_sge;
169
170 u32 inline_oob_size;
171 const void *inline_oob_data;
172
173 u32 flags;
174 u32 client_data_unit;
175};
176
177enum gdma_page_type {
178 GDMA_PAGE_TYPE_4K,
179};
180
181#define GDMA_INVALID_DMA_REGION 0
182
183struct gdma_mem_info {
184 struct device *dev;
185
186 dma_addr_t dma_handle;
187 void *virt_addr;
188 u64 length;
189
190
191 u64 gdma_region;
192};
193
194#define REGISTER_ATB_MST_MKEY_LOWER_SIZE 8
195
196struct gdma_dev {
197 struct gdma_context *gdma_context;
198
199 struct gdma_dev_id dev_id;
200
201 u32 pdid;
202 u32 doorbell;
203 u32 gpa_mkey;
204
205
206 void *driver_data;
207};
208
209#define MINIMUM_SUPPORTED_PAGE_SIZE PAGE_SIZE
210
211#define GDMA_CQE_SIZE 64
212#define GDMA_EQE_SIZE 16
213#define GDMA_MAX_SQE_SIZE 512
214#define GDMA_MAX_RQE_SIZE 256
215
216#define GDMA_COMP_DATA_SIZE 0x3C
217
218#define GDMA_EVENT_DATA_SIZE 0xC
219
220
221#define GDMA_WQE_BU_SIZE 32
222
223#define INVALID_PDID UINT_MAX
224#define INVALID_DOORBELL UINT_MAX
225#define INVALID_MEM_KEY UINT_MAX
226#define INVALID_QUEUE_ID UINT_MAX
227#define INVALID_PCI_MSIX_INDEX UINT_MAX
228
229struct gdma_comp {
230 u32 cqe_data[GDMA_COMP_DATA_SIZE / 4];
231 u32 wq_num;
232 bool is_sq;
233};
234
235struct gdma_event {
236 u32 details[GDMA_EVENT_DATA_SIZE / 4];
237 u8 type;
238};
239
240struct gdma_queue;
241
242struct mana_eq {
243 struct gdma_queue *eq;
244};
245
246typedef void gdma_eq_callback(void *context, struct gdma_queue *q,
247 struct gdma_event *e);
248
249typedef void gdma_cq_callback(void *context, struct gdma_queue *q);
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265struct gdma_queue {
266 struct gdma_dev *gdma_dev;
267
268 enum gdma_queue_type type;
269 u32 id;
270
271 struct gdma_mem_info mem_info;
272
273 void *queue_mem_ptr;
274 u32 queue_size;
275
276 bool monitor_avl_buf;
277
278 u32 head;
279 u32 tail;
280
281
282 union {
283 struct {
284 bool disable_needed;
285
286 gdma_eq_callback *callback;
287 void *context;
288
289 unsigned int msix_index;
290
291 u32 log2_throttle_limit;
292 } eq;
293
294 struct {
295 gdma_cq_callback *callback;
296 void *context;
297
298 struct gdma_queue *parent;
299 } cq;
300 };
301};
302
303struct gdma_queue_spec {
304 enum gdma_queue_type type;
305 bool monitor_avl_buf;
306 unsigned int queue_size;
307
308
309 union {
310 struct {
311 gdma_eq_callback *callback;
312 void *context;
313
314 unsigned long log2_throttle_limit;
315 } eq;
316
317 struct {
318 gdma_cq_callback *callback;
319 void *context;
320
321 struct gdma_queue *parent_eq;
322
323 } cq;
324 };
325};
326
327struct gdma_irq_context {
328 void (*handler)(void *arg);
329 void *arg;
330};
331
332struct gdma_context {
333 struct device *dev;
334
335
336 unsigned int max_num_queues;
337 unsigned int max_num_msix;
338 unsigned int num_msix_usable;
339 struct gdma_resource msix_resource;
340 struct gdma_irq_context *irq_contexts;
341
342
343 unsigned int max_num_cqs;
344 struct gdma_queue **cq_table;
345
346
347 struct mutex eq_test_event_mutex;
348 struct completion eq_test_event;
349 u32 test_event_eq_id;
350
351 void __iomem *bar0_va;
352 void __iomem *shm_base;
353 void __iomem *db_page_base;
354 u32 db_page_size;
355
356
357 struct shm_channel shm_channel;
358
359
360 struct gdma_dev hwc;
361
362
363 struct gdma_dev mana;
364};
365
366#define MAX_NUM_GDMA_DEVICES 4
367
368static inline bool mana_gd_is_mana(struct gdma_dev *gd)
369{
370 return gd->dev_id.type == GDMA_DEVICE_MANA;
371}
372
373static inline bool mana_gd_is_hwc(struct gdma_dev *gd)
374{
375 return gd->dev_id.type == GDMA_DEVICE_HWC;
376}
377
378u8 *mana_gd_get_wqe_ptr(const struct gdma_queue *wq, u32 wqe_offset);
379u32 mana_gd_wq_avail_space(struct gdma_queue *wq);
380
381int mana_gd_test_eq(struct gdma_context *gc, struct gdma_queue *eq);
382
383int mana_gd_create_hwc_queue(struct gdma_dev *gd,
384 const struct gdma_queue_spec *spec,
385 struct gdma_queue **queue_ptr);
386
387int mana_gd_create_mana_eq(struct gdma_dev *gd,
388 const struct gdma_queue_spec *spec,
389 struct gdma_queue **queue_ptr);
390
391int mana_gd_create_mana_wq_cq(struct gdma_dev *gd,
392 const struct gdma_queue_spec *spec,
393 struct gdma_queue **queue_ptr);
394
395void mana_gd_destroy_queue(struct gdma_context *gc, struct gdma_queue *queue);
396
397int mana_gd_poll_cq(struct gdma_queue *cq, struct gdma_comp *comp, int num_cqe);
398
399void mana_gd_ring_cq(struct gdma_queue *cq, u8 arm_bit);
400
401struct gdma_wqe {
402 u32 reserved :24;
403 u32 last_vbytes :8;
404
405 union {
406 u32 flags;
407
408 struct {
409 u32 num_sge :8;
410 u32 inline_oob_size_div4:3;
411 u32 client_oob_in_sgl :1;
412 u32 reserved1 :4;
413 u32 client_data_unit :14;
414 u32 reserved2 :2;
415 };
416 };
417};
418
419#define INLINE_OOB_SMALL_SIZE 8
420#define INLINE_OOB_LARGE_SIZE 24
421
422#define MAX_TX_WQE_SIZE 512
423#define MAX_RX_WQE_SIZE 256
424
425struct gdma_cqe {
426 u32 cqe_data[GDMA_COMP_DATA_SIZE / 4];
427
428 union {
429 u32 as_uint32;
430
431 struct {
432 u32 wq_num : 24;
433 u32 is_sq : 1;
434 u32 reserved : 4;
435 u32 owner_bits : 3;
436 };
437 } cqe_info;
438};
439
440#define GDMA_CQE_OWNER_BITS 3
441
442#define GDMA_CQE_OWNER_MASK ((1 << GDMA_CQE_OWNER_BITS) - 1)
443
444#define SET_ARM_BIT 1
445
446#define GDMA_EQE_OWNER_BITS 3
447
448union gdma_eqe_info {
449 u32 as_uint32;
450
451 struct {
452 u32 type : 8;
453 u32 reserved1 : 8;
454 u32 client_id : 2;
455 u32 reserved2 : 11;
456 u32 owner_bits : 3;
457 };
458};
459
460#define GDMA_EQE_OWNER_MASK ((1 << GDMA_EQE_OWNER_BITS) - 1)
461#define INITIALIZED_OWNER_BIT(log2_num_entries) (1UL << (log2_num_entries))
462
463struct gdma_eqe {
464 u32 details[GDMA_EVENT_DATA_SIZE / 4];
465 u32 eqe_info;
466};
467
468#define GDMA_REG_DB_PAGE_OFFSET 8
469#define GDMA_REG_DB_PAGE_SIZE 0x10
470#define GDMA_REG_SHM_OFFSET 0x18
471
472struct gdma_posted_wqe_info {
473 u32 wqe_size_in_bu;
474};
475
476
477struct gdma_generate_test_event_req {
478 struct gdma_req_hdr hdr;
479 u32 queue_index;
480};
481
482
483enum {
484 GDMA_PROTOCOL_V1 = 1,
485 GDMA_PROTOCOL_FIRST = GDMA_PROTOCOL_V1,
486 GDMA_PROTOCOL_LAST = GDMA_PROTOCOL_V1,
487};
488
489#define GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT BIT(0)
490
491#define GDMA_DRV_CAP_FLAGS1 GDMA_DRV_CAP_FLAG_1_EQ_SHARING_MULTI_VPORT
492
493#define GDMA_DRV_CAP_FLAGS2 0
494
495#define GDMA_DRV_CAP_FLAGS3 0
496
497#define GDMA_DRV_CAP_FLAGS4 0
498
499struct gdma_verify_ver_req {
500 struct gdma_req_hdr hdr;
501
502
503 u64 protocol_ver_min;
504 u64 protocol_ver_max;
505
506
507 u64 gd_drv_cap_flags1;
508 u64 gd_drv_cap_flags2;
509 u64 gd_drv_cap_flags3;
510 u64 gd_drv_cap_flags4;
511
512
513 u64 drv_ver;
514 u32 os_type;
515 u32 reserved;
516 u32 os_ver_major;
517 u32 os_ver_minor;
518 u32 os_ver_build;
519 u32 os_ver_platform;
520 u64 reserved_2;
521 u8 os_ver_str1[128];
522 u8 os_ver_str2[128];
523 u8 os_ver_str3[128];
524 u8 os_ver_str4[128];
525};
526
527struct gdma_verify_ver_resp {
528 struct gdma_resp_hdr hdr;
529 u64 gdma_protocol_ver;
530 u64 pf_cap_flags1;
531 u64 pf_cap_flags2;
532 u64 pf_cap_flags3;
533 u64 pf_cap_flags4;
534};
535
536
537struct gdma_query_max_resources_resp {
538 struct gdma_resp_hdr hdr;
539 u32 status;
540 u32 max_sq;
541 u32 max_rq;
542 u32 max_cq;
543 u32 max_eq;
544 u32 max_db;
545 u32 max_mst;
546 u32 max_cq_mod_ctx;
547 u32 max_mod_cq;
548 u32 max_msix;
549};
550
551
552struct gdma_list_devices_resp {
553 struct gdma_resp_hdr hdr;
554 u32 num_of_devs;
555 u32 reserved;
556 struct gdma_dev_id devs[64];
557};
558
559
560struct gdma_register_device_resp {
561 struct gdma_resp_hdr hdr;
562 u32 pdid;
563 u32 gpa_mkey;
564 u32 db_id;
565};
566
567
568struct gdma_create_queue_req {
569 struct gdma_req_hdr hdr;
570 u32 type;
571 u32 reserved1;
572 u32 pdid;
573 u32 doolbell_id;
574 u64 gdma_region;
575 u32 reserved2;
576 u32 queue_size;
577 u32 log2_throttle_limit;
578 u32 eq_pci_msix_index;
579 u32 cq_mod_ctx_id;
580 u32 cq_parent_eq_id;
581 u8 rq_drop_on_overrun;
582 u8 rq_err_on_wqe_overflow;
583 u8 rq_chain_rec_wqes;
584 u8 sq_hw_db;
585 u32 reserved3;
586};
587
588struct gdma_create_queue_resp {
589 struct gdma_resp_hdr hdr;
590 u32 queue_index;
591};
592
593
594struct gdma_disable_queue_req {
595 struct gdma_req_hdr hdr;
596 u32 type;
597 u32 queue_index;
598 u32 alloc_res_id_on_creation;
599};
600
601
602struct gdma_create_dma_region_req {
603 struct gdma_req_hdr hdr;
604
605
606 u64 length;
607
608
609 u32 offset_in_page;
610
611
612 u32 gdma_page_type;
613
614
615 u32 page_count;
616
617
618
619
620
621 u32 page_addr_list_len;
622 u64 page_addr_list[];
623};
624
625struct gdma_create_dma_region_resp {
626 struct gdma_resp_hdr hdr;
627 u64 gdma_region;
628};
629
630
631struct gdma_dma_region_add_pages_req {
632 struct gdma_req_hdr hdr;
633
634 u64 gdma_region;
635
636 u32 page_addr_list_len;
637 u32 reserved3;
638
639 u64 page_addr_list[];
640};
641
642
643struct gdma_destroy_dma_region_req {
644 struct gdma_req_hdr hdr;
645
646 u64 gdma_region;
647};
648
649int mana_gd_verify_vf_version(struct pci_dev *pdev);
650
651int mana_gd_register_device(struct gdma_dev *gd);
652int mana_gd_deregister_device(struct gdma_dev *gd);
653
654int mana_gd_post_work_request(struct gdma_queue *wq,
655 const struct gdma_wqe_request *wqe_req,
656 struct gdma_posted_wqe_info *wqe_info);
657
658int mana_gd_post_and_ring(struct gdma_queue *queue,
659 const struct gdma_wqe_request *wqe,
660 struct gdma_posted_wqe_info *wqe_info);
661
662int mana_gd_alloc_res_map(u32 res_avail, struct gdma_resource *r);
663void mana_gd_free_res_map(struct gdma_resource *r);
664
665void mana_gd_wq_ring_doorbell(struct gdma_context *gc,
666 struct gdma_queue *queue);
667
668int mana_gd_alloc_memory(struct gdma_context *gc, unsigned int length,
669 struct gdma_mem_info *gmi);
670
671void mana_gd_free_memory(struct gdma_mem_info *gmi);
672
673int mana_gd_send_request(struct gdma_context *gc, u32 req_len, const void *req,
674 u32 resp_len, void *resp);
675#endif
676