1
2
3
4#include <net/xdp_sock_drv.h>
5#include "pool.h"
6#include "setup.h"
7#include "en/params.h"
8
9static int mlx5e_xsk_map_pool(struct mlx5e_priv *priv,
10 struct xsk_buff_pool *pool)
11{
12 struct device *dev = mlx5_core_dma_dev(priv->mdev);
13
14 return xsk_pool_dma_map(pool, dev, 0);
15}
16
17static void mlx5e_xsk_unmap_pool(struct mlx5e_priv *priv,
18 struct xsk_buff_pool *pool)
19{
20 return xsk_pool_dma_unmap(pool, 0);
21}
22
23static int mlx5e_xsk_get_pools(struct mlx5e_xsk *xsk)
24{
25 if (!xsk->pools) {
26 xsk->pools = kcalloc(MLX5E_MAX_NUM_CHANNELS,
27 sizeof(*xsk->pools), GFP_KERNEL);
28 if (unlikely(!xsk->pools))
29 return -ENOMEM;
30 }
31
32 xsk->refcnt++;
33 xsk->ever_used = true;
34
35 return 0;
36}
37
38static void mlx5e_xsk_put_pools(struct mlx5e_xsk *xsk)
39{
40 if (!--xsk->refcnt) {
41 kfree(xsk->pools);
42 xsk->pools = NULL;
43 }
44}
45
46static int mlx5e_xsk_add_pool(struct mlx5e_xsk *xsk, struct xsk_buff_pool *pool, u16 ix)
47{
48 int err;
49
50 err = mlx5e_xsk_get_pools(xsk);
51 if (unlikely(err))
52 return err;
53
54 xsk->pools[ix] = pool;
55 return 0;
56}
57
58static void mlx5e_xsk_remove_pool(struct mlx5e_xsk *xsk, u16 ix)
59{
60 xsk->pools[ix] = NULL;
61
62 mlx5e_xsk_put_pools(xsk);
63}
64
65static bool mlx5e_xsk_is_pool_sane(struct xsk_buff_pool *pool)
66{
67 return xsk_pool_get_headroom(pool) <= 0xffff &&
68 xsk_pool_get_chunk_size(pool) <= 0xffff;
69}
70
71void mlx5e_build_xsk_param(struct xsk_buff_pool *pool, struct mlx5e_xsk_param *xsk)
72{
73 xsk->headroom = xsk_pool_get_headroom(pool);
74 xsk->chunk_size = xsk_pool_get_chunk_size(pool);
75}
76
77static int mlx5e_xsk_enable_locked(struct mlx5e_priv *priv,
78 struct xsk_buff_pool *pool, u16 ix)
79{
80 struct mlx5e_params *params = &priv->channels.params;
81 struct mlx5e_xsk_param xsk;
82 struct mlx5e_channel *c;
83 int err;
84
85 if (unlikely(mlx5e_xsk_get_pool(&priv->channels.params, &priv->xsk, ix)))
86 return -EBUSY;
87
88 if (unlikely(!mlx5e_xsk_is_pool_sane(pool)))
89 return -EINVAL;
90
91 err = mlx5e_xsk_map_pool(priv, pool);
92 if (unlikely(err))
93 return err;
94
95 err = mlx5e_xsk_add_pool(&priv->xsk, pool, ix);
96 if (unlikely(err))
97 goto err_unmap_pool;
98
99 mlx5e_build_xsk_param(pool, &xsk);
100
101 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) {
102
103 goto validate_closed;
104 }
105
106 if (!params->xdp_prog) {
107
108
109
110 goto validate_closed;
111 }
112
113 c = priv->channels.c[ix];
114
115 err = mlx5e_open_xsk(priv, params, &xsk, pool, c);
116 if (unlikely(err))
117 goto err_remove_pool;
118
119 mlx5e_activate_xsk(c);
120
121
122
123
124
125 err = mlx5e_xsk_redirect_rqt_to_channel(priv, priv->channels.c[ix]);
126 if (unlikely(err))
127 goto err_deactivate;
128
129 return 0;
130
131err_deactivate:
132 mlx5e_deactivate_xsk(c);
133 mlx5e_close_xsk(c);
134
135err_remove_pool:
136 mlx5e_xsk_remove_pool(&priv->xsk, ix);
137
138err_unmap_pool:
139 mlx5e_xsk_unmap_pool(priv, pool);
140
141 return err;
142
143validate_closed:
144
145
146
147 if (!mlx5e_validate_xsk_param(params, &xsk, priv->mdev)) {
148 err = -EINVAL;
149 goto err_remove_pool;
150 }
151
152 return 0;
153}
154
155static int mlx5e_xsk_disable_locked(struct mlx5e_priv *priv, u16 ix)
156{
157 struct xsk_buff_pool *pool = mlx5e_xsk_get_pool(&priv->channels.params,
158 &priv->xsk, ix);
159 struct mlx5e_channel *c;
160
161 if (unlikely(!pool))
162 return -EINVAL;
163
164 if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
165 goto remove_pool;
166
167
168 if (!priv->channels.params.xdp_prog)
169 goto remove_pool;
170
171 c = priv->channels.c[ix];
172 mlx5e_xsk_redirect_rqt_to_drop(priv, ix);
173 mlx5e_deactivate_xsk(c);
174 mlx5e_close_xsk(c);
175
176remove_pool:
177 mlx5e_xsk_remove_pool(&priv->xsk, ix);
178 mlx5e_xsk_unmap_pool(priv, pool);
179
180 return 0;
181}
182
183static int mlx5e_xsk_enable_pool(struct mlx5e_priv *priv, struct xsk_buff_pool *pool,
184 u16 ix)
185{
186 int err;
187
188 mutex_lock(&priv->state_lock);
189 err = mlx5e_xsk_enable_locked(priv, pool, ix);
190 mutex_unlock(&priv->state_lock);
191
192 return err;
193}
194
195static int mlx5e_xsk_disable_pool(struct mlx5e_priv *priv, u16 ix)
196{
197 int err;
198
199 mutex_lock(&priv->state_lock);
200 err = mlx5e_xsk_disable_locked(priv, ix);
201 mutex_unlock(&priv->state_lock);
202
203 return err;
204}
205
206int mlx5e_xsk_setup_pool(struct net_device *dev, struct xsk_buff_pool *pool, u16 qid)
207{
208 struct mlx5e_priv *priv = netdev_priv(dev);
209 struct mlx5e_params *params = &priv->channels.params;
210 u16 ix;
211
212 if (unlikely(!mlx5e_qid_get_ch_if_in_group(params, qid, MLX5E_RQ_GROUP_XSK, &ix)))
213 return -EINVAL;
214
215 return pool ? mlx5e_xsk_enable_pool(priv, pool, ix) :
216 mlx5e_xsk_disable_pool(priv, ix);
217}
218