1
2
3
4
5#ifndef RTE_PMD_MLX5_COMMON_H_
6#define RTE_PMD_MLX5_COMMON_H_
7
8#include <stdio.h>
9
10#include <rte_pci.h>
11#include <rte_bus_pci.h>
12#include <rte_debug.h>
13#include <rte_atomic.h>
14#include <rte_rwlock.h>
15#include <rte_log.h>
16#include <rte_kvargs.h>
17#include <rte_devargs.h>
18#include <rte_bitops.h>
19#include <rte_lcore.h>
20#include <rte_spinlock.h>
21#include <rte_os_shim.h>
22
23#include "mlx5_prm.h"
24#include "mlx5_devx_cmds.h"
25#include "mlx5_common_os.h"
26#include "mlx5_common_mr.h"
27
28
29#define MLX5_PCI_DRIVER_NAME "mlx5_pci"
30#define MLX5_AUXILIARY_DRIVER_NAME "mlx5_auxiliary"
31
32
33#define BITFIELD_DECLARE(bf, type, size) \
34 type bf[(((size_t)(size) / (sizeof(type) * CHAR_BIT)) + \
35 !!((size_t)(size) % (sizeof(type) * CHAR_BIT)))]
36#define BITFIELD_DEFINE(bf, type, size) \
37 BITFIELD_DECLARE((bf), type, (size)) = { 0 }
38#define BITFIELD_SET(bf, b) \
39 (void)((bf)[((b) / (sizeof((bf)[0]) * CHAR_BIT))] |= \
40 ((size_t)1 << ((b) % (sizeof((bf)[0]) * CHAR_BIT))))
41#define BITFIELD_RESET(bf, b) \
42 (void)((bf)[((b) / (sizeof((bf)[0]) * CHAR_BIT))] &= \
43 ~((size_t)1 << ((b) % (sizeof((bf)[0]) * CHAR_BIT))))
44#define BITFIELD_ISSET(bf, b) \
45 !!(((bf)[((b) / (sizeof((bf)[0]) * CHAR_BIT))] & \
46 ((size_t)1 << ((b) % (sizeof((bf)[0]) * CHAR_BIT)))))
47
48
49
50
51
52#define PMD_DRV_LOG_STRIP(a, b) a
53#define PMD_DRV_LOG_OPAREN (
54#define PMD_DRV_LOG_CPAREN )
55#define PMD_DRV_LOG_COMMA ,
56
57
58static inline const char *
59pmd_drv_log_basename(const char *s)
60{
61 const char *n = s;
62
63 while (*n)
64 if (*(n++) == '/')
65 s = n;
66 return s;
67}
68
69#define PMD_DRV_LOG___(level, type, name, ...) \
70 rte_log(RTE_LOG_ ## level, \
71 type, \
72 RTE_FMT(name ": " \
73 RTE_FMT_HEAD(__VA_ARGS__,), \
74 RTE_FMT_TAIL(__VA_ARGS__,)))
75
76#ifdef RTE_LIBRTE_MLX5_DEBUG
77
78#define PMD_DRV_LOG__(level, type, name, ...) \
79 PMD_DRV_LOG___(level, type, name, "%s:%u: %s(): " __VA_ARGS__)
80#define PMD_DRV_LOG_(level, type, name, s, ...) \
81 PMD_DRV_LOG__(level, type, name,\
82 s "\n" PMD_DRV_LOG_COMMA \
83 pmd_drv_log_basename(__FILE__) PMD_DRV_LOG_COMMA \
84 __LINE__ PMD_DRV_LOG_COMMA \
85 __func__, \
86 __VA_ARGS__)
87
88#else
89#define PMD_DRV_LOG__(level, type, name, ...) \
90 PMD_DRV_LOG___(level, type, name, __VA_ARGS__)
91#define PMD_DRV_LOG_(level, type, name, s, ...) \
92 PMD_DRV_LOG__(level, type, name, s "\n", __VA_ARGS__)
93
94#endif
95
96
97#ifdef RTE_LIBRTE_MLX5_DEBUG
98
99#define MLX5_ASSERT(exp) RTE_VERIFY(exp)
100#define claim_zero(...) MLX5_ASSERT((__VA_ARGS__) == 0)
101#define claim_nonzero(...) MLX5_ASSERT((__VA_ARGS__) != 0)
102
103#else
104
105#define MLX5_ASSERT(exp) RTE_ASSERT(exp)
106#define claim_zero(...) (__VA_ARGS__)
107#define claim_nonzero(...) (__VA_ARGS__)
108
109#endif
110
111
112#define MKSTR(name, ...) \
113 int mkstr_size_##name = snprintf(NULL, 0, "" __VA_ARGS__); \
114 char name[mkstr_size_##name + 1]; \
115 \
116 memset(name, 0, mkstr_size_##name + 1); \
117 snprintf(name, sizeof(name), "" __VA_ARGS__)
118
119enum {
120 PCI_VENDOR_ID_MELLANOX = 0x15b3,
121};
122
123enum {
124 PCI_DEVICE_ID_MELLANOX_CONNECTX4 = 0x1013,
125 PCI_DEVICE_ID_MELLANOX_CONNECTX4VF = 0x1014,
126 PCI_DEVICE_ID_MELLANOX_CONNECTX4LX = 0x1015,
127 PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF = 0x1016,
128 PCI_DEVICE_ID_MELLANOX_CONNECTX5 = 0x1017,
129 PCI_DEVICE_ID_MELLANOX_CONNECTX5VF = 0x1018,
130 PCI_DEVICE_ID_MELLANOX_CONNECTX5EX = 0x1019,
131 PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF = 0x101a,
132 PCI_DEVICE_ID_MELLANOX_CONNECTX5BF = 0xa2d2,
133 PCI_DEVICE_ID_MELLANOX_CONNECTX5BFVF = 0xa2d3,
134 PCI_DEVICE_ID_MELLANOX_CONNECTX6 = 0x101b,
135 PCI_DEVICE_ID_MELLANOX_CONNECTX6VF = 0x101c,
136 PCI_DEVICE_ID_MELLANOX_CONNECTX6DX = 0x101d,
137 PCI_DEVICE_ID_MELLANOX_CONNECTXVF = 0x101e,
138 PCI_DEVICE_ID_MELLANOX_CONNECTX6DXBF = 0xa2d6,
139 PCI_DEVICE_ID_MELLANOX_CONNECTX6LX = 0x101f,
140 PCI_DEVICE_ID_MELLANOX_CONNECTX7 = 0x1021,
141 PCI_DEVICE_ID_MELLANOX_CONNECTX7BF = 0Xa2dc,
142};
143
144
145#define MLX5_MAX_UC_MAC_ADDRESSES 128
146
147#define MLX5_MAX_MC_MAC_ADDRESSES 128
148
149#define MLX5_MAX_MAC_ADDRESSES \
150 (MLX5_MAX_UC_MAC_ADDRESSES + MLX5_MAX_MC_MAC_ADDRESSES)
151
152
153enum mlx5_nl_phys_port_name_type {
154 MLX5_PHYS_PORT_NAME_TYPE_NOTSET = 0,
155 MLX5_PHYS_PORT_NAME_TYPE_LEGACY,
156 MLX5_PHYS_PORT_NAME_TYPE_UPLINK,
157 MLX5_PHYS_PORT_NAME_TYPE_PFVF,
158 MLX5_PHYS_PORT_NAME_TYPE_PFHPF,
159 MLX5_PHYS_PORT_NAME_TYPE_PFSF,
160 MLX5_PHYS_PORT_NAME_TYPE_UNKNOWN,
161};
162
163
164struct mlx5_switch_info {
165 uint32_t master:1;
166 uint32_t representor:1;
167 enum mlx5_nl_phys_port_name_type name_type;
168 int32_t ctrl_num;
169 int32_t pf_num;
170 int32_t port_name;
171 uint64_t switch_id;
172};
173
174
175enum mlx5_cqe_status {
176 MLX5_CQE_STATUS_SW_OWN = -1,
177 MLX5_CQE_STATUS_HW_OWN = -2,
178 MLX5_CQE_STATUS_ERR = -3,
179};
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194static __rte_always_inline enum mlx5_cqe_status
195check_cqe(volatile struct mlx5_cqe *cqe, const uint16_t cqes_n,
196 const uint16_t ci)
197{
198 const uint16_t idx = ci & cqes_n;
199 const uint8_t op_own = cqe->op_own;
200 const uint8_t op_owner = MLX5_CQE_OWNER(op_own);
201 const uint8_t op_code = MLX5_CQE_OPCODE(op_own);
202
203 if (unlikely((op_owner != (!!(idx))) || (op_code == MLX5_CQE_INVALID)))
204 return MLX5_CQE_STATUS_HW_OWN;
205 rte_io_rmb();
206 if (unlikely(op_code == MLX5_CQE_RESP_ERR ||
207 op_code == MLX5_CQE_REQ_ERR))
208 return MLX5_CQE_STATUS_ERR;
209 return MLX5_CQE_STATUS_SW_OWN;
210}
211
212
213
214
215
216
217
218
219
220
221
222
223int mlx5_dev_to_pci_str(const struct rte_device *dev, char *addr, size_t size);
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238__rte_internal
239int mlx5_get_pci_addr(const char *dev_path, struct rte_pci_addr *pci_addr);
240
241
242
243
244
245
246
247
248
249
250
251
252
253__rte_internal
254int mlx5_get_ifname_sysfs(const char *ibdev_path, char *ifname);
255
256__rte_internal
257int mlx5_auxiliary_get_child_name(const char *dev, const char *node,
258 char *child, size_t size);
259
260enum mlx5_class {
261 MLX5_CLASS_INVALID,
262 MLX5_CLASS_ETH = RTE_BIT64(0),
263 MLX5_CLASS_VDPA = RTE_BIT64(1),
264 MLX5_CLASS_REGEX = RTE_BIT64(2),
265 MLX5_CLASS_COMPRESS = RTE_BIT64(3),
266 MLX5_CLASS_CRYPTO = RTE_BIT64(4),
267};
268
269#define MLX5_DBR_SIZE RTE_CACHE_LINE_SIZE
270
271
272struct mlx5_devx_obj {
273 void *obj;
274 int id;
275};
276
277
278struct mlx5_klm {
279 uint32_t byte_count;
280 uint32_t mkey;
281 uint64_t address;
282};
283
284
285struct mlx5_kvargs_ctrl {
286 struct rte_kvargs *kvlist;
287 bool is_used[RTE_KVARGS_MAX];
288};
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310__rte_internal
311int
312mlx5_kvargs_process(struct mlx5_kvargs_ctrl *mkvlist, const char *const keys[],
313 arg_handler_t handler, void *opaque_arg);
314
315
316struct mlx5_uar_data {
317 uint64_t *db;
318
319#ifndef RTE_ARCH_64
320 rte_spinlock_t *sl_p;
321
322#endif
323};
324
325
326struct mlx5_uar {
327 struct mlx5_uar_data bf_db;
328 struct mlx5_uar_data cq_db;
329 void *obj;
330 bool dbnc;
331#ifndef RTE_ARCH_64
332 rte_spinlock_t bf_sl;
333 rte_spinlock_t cq_sl;
334
335#endif
336};
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352static __rte_always_inline void
353mlx5_doorbell_ring(struct mlx5_uar_data *uar, uint64_t val, uint32_t index,
354 volatile uint32_t *db_rec, bool flash)
355{
356 rte_io_wmb();
357 *db_rec = rte_cpu_to_be_32(index);
358
359 rte_wmb();
360#ifdef RTE_ARCH_64
361 *uar->db = val;
362#else
363 rte_spinlock_lock(uar->sl_p);
364 *(volatile uint32_t *)uar->db = val;
365 rte_io_wmb();
366 *((volatile uint32_t *)uar->db + 1) = val >> 32;
367 rte_spinlock_unlock(uar->sl_p);
368#endif
369 if (flash)
370 rte_wmb();
371}
372
373
374
375
376
377
378
379
380
381
382
383
384static inline uint16_t
385mlx5_db_map_type_get(off_t uar_mmap_offset, size_t page_size)
386{
387 off_t cmd = uar_mmap_offset / page_size;
388
389 cmd >>= MLX5_UAR_MMAP_CMD_SHIFT;
390 cmd &= MLX5_UAR_MMAP_CMD_MASK;
391 if (cmd == MLX5_MMAP_GET_NC_PAGES_CMD)
392 return 1;
393 return 0;
394}
395
396__rte_internal
397void mlx5_translate_port_name(const char *port_name_in,
398 struct mlx5_switch_info *port_info_out);
399void mlx5_glue_constructor(void);
400extern uint8_t haswell_broadwell_cpu;
401
402__rte_internal
403void mlx5_common_init(void);
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446struct mlx5_common_dev_config {
447 struct mlx5_hca_attr hca_attr;
448 int dbnc;
449 int device_fd;
450 int pd_handle;
451 unsigned int devx:1;
452 unsigned int sys_mem_en:1;
453 unsigned int mr_mempool_reg_en:1;
454
455 unsigned int mr_ext_memseg_en:1;
456
457};
458
459struct mlx5_common_device {
460 struct rte_device *dev;
461 TAILQ_ENTRY(mlx5_common_device) next;
462 uint32_t classes_loaded;
463 void *ctx;
464 void *pd;
465 uint32_t pdn;
466 struct mlx5_mr_share_cache mr_scache;
467 struct mlx5_common_dev_config config;
468};
469
470
471
472
473
474
475
476
477
478
479
480static inline bool
481mlx5_imported_pd_and_ctx(struct mlx5_common_device *cdev)
482{
483 return cdev->config.device_fd != MLX5_ARG_UNSET &&
484 cdev->config.pd_handle != MLX5_ARG_UNSET;
485}
486
487
488
489
490typedef int (mlx5_class_driver_probe_t)(struct mlx5_common_device *cdev,
491 struct mlx5_kvargs_ctrl *mkvlist);
492
493
494
495
496typedef int (mlx5_class_driver_remove_t)(struct mlx5_common_device *cdev);
497
498
499#define MLX5_DRV_PROBE_AGAIN 0x0004
500
501
502
503
504struct mlx5_class_driver {
505 TAILQ_ENTRY(mlx5_class_driver) next;
506 enum mlx5_class drv_class;
507 const char *name;
508 mlx5_class_driver_probe_t *probe;
509 mlx5_class_driver_remove_t *remove;
510 const struct rte_pci_id *id_table;
511 uint32_t probe_again:1;
512
513 uint32_t intr_lsc:1;
514 uint32_t intr_rmv:1;
515};
516
517
518
519
520
521
522
523
524__rte_internal
525void
526mlx5_class_driver_register(struct mlx5_class_driver *driver);
527
528
529
530
531
532
533
534
535
536
537
538__rte_internal
539bool
540mlx5_dev_is_pci(const struct rte_device *dev);
541
542
543
544
545
546
547
548
549
550
551
552__rte_internal
553bool
554mlx5_dev_is_vf_pci(struct rte_pci_device *pci_dev);
555
556__rte_internal
557int
558mlx5_dev_mempool_subscribe(struct mlx5_common_device *cdev);
559
560__rte_internal
561void
562mlx5_dev_mempool_unregister(struct mlx5_common_device *cdev,
563 struct rte_mempool *mp);
564
565__rte_internal
566int
567mlx5_devx_uar_prepare(struct mlx5_common_device *cdev, struct mlx5_uar *uar);
568
569__rte_internal
570void
571mlx5_devx_uar_release(struct mlx5_uar *uar);
572
573
574
575int mlx5_os_open_device(struct mlx5_common_device *cdev, uint32_t classes);
576int mlx5_os_pd_prepare(struct mlx5_common_device *cdev);
577int mlx5_os_pd_release(struct mlx5_common_device *cdev);
578int mlx5_os_remote_pd_and_ctx_validate(struct mlx5_common_dev_config *config);
579
580
581struct mlx5_pmd_wrapped_mr {
582 uint32_t lkey;
583 void *addr;
584 size_t len;
585 void *obj;
586 void *imkey;
587};
588
589__rte_internal
590int
591mlx5_os_wrapped_mkey_create(void *ctx, void *pd, uint32_t pdn, void *addr,
592 size_t length, struct mlx5_pmd_wrapped_mr *pmd_mr);
593
594__rte_internal
595void
596mlx5_os_wrapped_mkey_destroy(struct mlx5_pmd_wrapped_mr *pmd_mr);
597
598#endif
599