1
2
3
4
5
6#ifndef RTE_PMD_MLX5_COMMON_MR_H_
7#define RTE_PMD_MLX5_COMMON_MR_H_
8
9#include <stddef.h>
10#include <stdint.h>
11#include <sys/queue.h>
12
13
14#include <rte_rwlock.h>
15#include <rte_bitmap.h>
16#include <rte_memory.h>
17
18#include "mlx5_glue.h"
19#include "mlx5_common_mp.h"
20
21
22#define MLX5_MR_CACHE_N 8
23#define MLX5_MR_BTREE_CACHE_N 256
24
25
26struct mlx5_pmd_mr {
27 uint32_t lkey;
28 void *addr;
29 size_t len;
30 void *obj;
31 struct mlx5_devx_obj *mkey;
32};
33
34
35
36
37typedef int (*mlx5_reg_mr_t)(void *pd, void *addr, size_t length,
38 struct mlx5_pmd_mr *pmd_mr);
39typedef void (*mlx5_dereg_mr_t)(struct mlx5_pmd_mr *pmd_mr);
40
41
42struct mlx5_mr {
43 LIST_ENTRY(mlx5_mr) mr;
44 struct mlx5_pmd_mr pmd_mr;
45 const struct rte_memseg_list *msl;
46 int ms_base_idx;
47 int ms_n;
48 uint32_t ms_bmp_n;
49 struct rte_bitmap *ms_bmp;
50};
51
52
53struct mr_cache_entry {
54 uintptr_t start;
55 uintptr_t end;
56 uint32_t lkey;
57} __rte_packed;
58
59
60struct mlx5_mr_btree {
61 uint16_t len;
62 uint16_t size;
63 int overflow;
64 struct mr_cache_entry (*table)[];
65} __rte_packed;
66
67
68struct mlx5_mr_ctrl {
69 uint32_t *dev_gen_ptr;
70 uint32_t cur_gen;
71 uint16_t mru;
72 uint16_t head;
73 struct mr_cache_entry cache[MLX5_MR_CACHE_N];
74 struct mlx5_mr_btree cache_bh;
75} __rte_packed;
76
77LIST_HEAD(mlx5_mr_list, mlx5_mr);
78
79
80struct mlx5_mr_share_cache {
81 uint32_t dev_gen;
82 rte_rwlock_t rwlock;
83 struct mlx5_mr_btree cache;
84 struct mlx5_mr_list mr_list;
85 struct mlx5_mr_list mr_free_list;
86 mlx5_reg_mr_t reg_mr_cb;
87 mlx5_dereg_mr_t dereg_mr_cb;
88} __rte_packed;
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107static __rte_always_inline uint32_t
108mlx5_mr_lookup_lkey(struct mr_cache_entry *lkp_tbl, uint16_t *cached_idx,
109 uint16_t n, uintptr_t addr)
110{
111 uint16_t idx;
112
113 if (likely(addr >= lkp_tbl[*cached_idx].start &&
114 addr < lkp_tbl[*cached_idx].end))
115 return lkp_tbl[*cached_idx].lkey;
116 for (idx = 0; idx < n && lkp_tbl[idx].start != 0; ++idx) {
117 if (addr >= lkp_tbl[idx].start &&
118 addr < lkp_tbl[idx].end) {
119
120 *cached_idx = idx;
121 return lkp_tbl[idx].lkey;
122 }
123 }
124 return UINT32_MAX;
125}
126
127__rte_internal
128int mlx5_mr_btree_init(struct mlx5_mr_btree *bt, int n, int socket);
129__rte_internal
130void mlx5_mr_btree_free(struct mlx5_mr_btree *bt);
131__rte_internal
132void mlx5_mr_btree_dump(struct mlx5_mr_btree *bt __rte_unused);
133__rte_internal
134uint32_t mlx5_mr_addr2mr_bh(void *pd, struct mlx5_mp_id *mp_id,
135 struct mlx5_mr_share_cache *share_cache,
136 struct mlx5_mr_ctrl *mr_ctrl,
137 uintptr_t addr, unsigned int mr_ext_memseg_en);
138__rte_internal
139void mlx5_mr_release_cache(struct mlx5_mr_share_cache *mr_cache);
140__rte_internal
141void mlx5_mr_dump_cache(struct mlx5_mr_share_cache *share_cache __rte_unused);
142__rte_internal
143void mlx5_mr_rebuild_cache(struct mlx5_mr_share_cache *share_cache);
144__rte_internal
145void mlx5_mr_flush_local_cache(struct mlx5_mr_ctrl *mr_ctrl);
146__rte_internal
147int
148mlx5_mr_insert_cache(struct mlx5_mr_share_cache *share_cache,
149 struct mlx5_mr *mr);
150__rte_internal
151uint32_t
152mlx5_mr_lookup_cache(struct mlx5_mr_share_cache *share_cache,
153 struct mr_cache_entry *entry, uintptr_t addr);
154__rte_internal
155struct mlx5_mr *
156mlx5_mr_lookup_list(struct mlx5_mr_share_cache *share_cache,
157 struct mr_cache_entry *entry, uintptr_t addr);
158__rte_internal
159struct mlx5_mr *
160mlx5_create_mr_ext(void *pd, uintptr_t addr, size_t len, int socket_id,
161 mlx5_reg_mr_t reg_mr_cb);
162__rte_internal
163uint32_t
164mlx5_mr_create_primary(void *pd,
165 struct mlx5_mr_share_cache *share_cache,
166 struct mr_cache_entry *entry, uintptr_t addr,
167 unsigned int mr_ext_memseg_en);
168__rte_internal
169int
170mlx5_common_verbs_reg_mr(void *pd, void *addr, size_t length,
171 struct mlx5_pmd_mr *pmd_mr);
172__rte_internal
173void
174mlx5_common_verbs_dereg_mr(struct mlx5_pmd_mr *pmd_mr);
175
176__rte_internal
177void
178mlx5_mr_free(struct mlx5_mr *mr, mlx5_dereg_mr_t dereg_mr_cb);
179#endif
180