1
2
3
4#include "mlx5_core.h"
5#include "ipsec_offload.h"
6#include "lib/mlx5.h"
7#include "en_accel/ipsec_fs.h"
8
9#define MLX5_IPSEC_DEV_BASIC_CAPS (MLX5_ACCEL_IPSEC_CAP_DEVICE | MLX5_ACCEL_IPSEC_CAP_IPV6 | \
10 MLX5_ACCEL_IPSEC_CAP_LSO)
11
12struct mlx5_ipsec_sa_ctx {
13 struct rhash_head hash;
14 u32 enc_key_id;
15 u32 ipsec_obj_id;
16
17 struct mlx5_core_dev *dev;
18 struct mlx5_ipsec_esp_xfrm *mxfrm;
19};
20
21struct mlx5_ipsec_esp_xfrm {
22
23 struct mlx5_ipsec_sa_ctx *sa_ctx;
24 struct mutex lock;
25 struct mlx5_accel_esp_xfrm accel_xfrm;
26};
27
28static u32 mlx5_ipsec_offload_device_caps(struct mlx5_core_dev *mdev)
29{
30 u32 caps = MLX5_IPSEC_DEV_BASIC_CAPS;
31
32 if (!mlx5_is_ipsec_device(mdev))
33 return 0;
34
35 if (!MLX5_CAP_FLOWTABLE_NIC_TX(mdev, ipsec_encrypt) ||
36 !MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ipsec_decrypt))
37 return 0;
38
39 if (MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_encrypt) &&
40 MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_decrypt))
41 caps |= MLX5_ACCEL_IPSEC_CAP_ESP;
42
43 if (MLX5_CAP_IPSEC(mdev, ipsec_esn)) {
44 caps |= MLX5_ACCEL_IPSEC_CAP_ESN;
45 caps |= MLX5_ACCEL_IPSEC_CAP_TX_IV_IS_ESN;
46 }
47
48
49
50
51
52 WARN_ON_ONCE(MLX5_CAP_IPSEC(mdev, log_max_ipsec_offload) > 24);
53 return caps;
54}
55
56static int
57mlx5_ipsec_offload_esp_validate_xfrm_attrs(struct mlx5_core_dev *mdev,
58 const struct mlx5_accel_esp_xfrm_attrs *attrs)
59{
60 if (attrs->replay_type != MLX5_ACCEL_ESP_REPLAY_NONE) {
61 mlx5_core_err(mdev, "Cannot offload xfrm states with anti replay (replay_type = %d)\n",
62 attrs->replay_type);
63 return -EOPNOTSUPP;
64 }
65
66 if (attrs->keymat_type != MLX5_ACCEL_ESP_KEYMAT_AES_GCM) {
67 mlx5_core_err(mdev, "Only aes gcm keymat is supported (keymat_type = %d)\n",
68 attrs->keymat_type);
69 return -EOPNOTSUPP;
70 }
71
72 if (attrs->keymat.aes_gcm.iv_algo !=
73 MLX5_ACCEL_ESP_AES_GCM_IV_ALGO_SEQ) {
74 mlx5_core_err(mdev, "Only iv sequence algo is supported (iv_algo = %d)\n",
75 attrs->keymat.aes_gcm.iv_algo);
76 return -EOPNOTSUPP;
77 }
78
79 if (attrs->keymat.aes_gcm.key_len != 128 &&
80 attrs->keymat.aes_gcm.key_len != 256) {
81 mlx5_core_err(mdev, "Cannot offload xfrm states with key length other than 128/256 bit (key length = %d)\n",
82 attrs->keymat.aes_gcm.key_len);
83 return -EOPNOTSUPP;
84 }
85
86 if ((attrs->flags & MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED) &&
87 !MLX5_CAP_IPSEC(mdev, ipsec_esn)) {
88 mlx5_core_err(mdev, "Cannot offload xfrm states with ESN triggered\n");
89 return -EOPNOTSUPP;
90 }
91
92 return 0;
93}
94
95static struct mlx5_accel_esp_xfrm *
96mlx5_ipsec_offload_esp_create_xfrm(struct mlx5_core_dev *mdev,
97 const struct mlx5_accel_esp_xfrm_attrs *attrs,
98 u32 flags)
99{
100 struct mlx5_ipsec_esp_xfrm *mxfrm;
101 int err = 0;
102
103 err = mlx5_ipsec_offload_esp_validate_xfrm_attrs(mdev, attrs);
104 if (err)
105 return ERR_PTR(err);
106
107 mxfrm = kzalloc(sizeof(*mxfrm), GFP_KERNEL);
108 if (!mxfrm)
109 return ERR_PTR(-ENOMEM);
110
111 mutex_init(&mxfrm->lock);
112 memcpy(&mxfrm->accel_xfrm.attrs, attrs,
113 sizeof(mxfrm->accel_xfrm.attrs));
114
115 return &mxfrm->accel_xfrm;
116}
117
118static void mlx5_ipsec_offload_esp_destroy_xfrm(struct mlx5_accel_esp_xfrm *xfrm)
119{
120 struct mlx5_ipsec_esp_xfrm *mxfrm = container_of(xfrm, struct mlx5_ipsec_esp_xfrm,
121 accel_xfrm);
122
123
124 WARN_ON(mxfrm->sa_ctx);
125 kfree(mxfrm);
126}
127
128struct mlx5_ipsec_obj_attrs {
129 const struct aes_gcm_keymat *aes_gcm;
130 u32 accel_flags;
131 u32 esn_msb;
132 u32 enc_key_id;
133};
134
135static int mlx5_create_ipsec_obj(struct mlx5_core_dev *mdev,
136 struct mlx5_ipsec_obj_attrs *attrs,
137 u32 *ipsec_id)
138{
139 const struct aes_gcm_keymat *aes_gcm = attrs->aes_gcm;
140 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
141 u32 in[MLX5_ST_SZ_DW(create_ipsec_obj_in)] = {};
142 void *obj, *salt_p, *salt_iv_p;
143 int err;
144
145 obj = MLX5_ADDR_OF(create_ipsec_obj_in, in, ipsec_object);
146
147
148 salt_p = MLX5_ADDR_OF(ipsec_obj, obj, salt);
149 memcpy(salt_p, &aes_gcm->salt, sizeof(aes_gcm->salt));
150
151 switch (aes_gcm->icv_len) {
152 case 64:
153 MLX5_SET(ipsec_obj, obj, icv_length,
154 MLX5_IPSEC_OBJECT_ICV_LEN_8B);
155 break;
156 case 96:
157 MLX5_SET(ipsec_obj, obj, icv_length,
158 MLX5_IPSEC_OBJECT_ICV_LEN_12B);
159 break;
160 case 128:
161 MLX5_SET(ipsec_obj, obj, icv_length,
162 MLX5_IPSEC_OBJECT_ICV_LEN_16B);
163 break;
164 default:
165 return -EINVAL;
166 }
167 salt_iv_p = MLX5_ADDR_OF(ipsec_obj, obj, implicit_iv);
168 memcpy(salt_iv_p, &aes_gcm->seq_iv, sizeof(aes_gcm->seq_iv));
169
170 if (attrs->accel_flags & MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED) {
171 MLX5_SET(ipsec_obj, obj, esn_en, 1);
172 MLX5_SET(ipsec_obj, obj, esn_msb, attrs->esn_msb);
173 if (attrs->accel_flags & MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP)
174 MLX5_SET(ipsec_obj, obj, esn_overlap, 1);
175 }
176
177 MLX5_SET(ipsec_obj, obj, dekn, attrs->enc_key_id);
178
179
180 MLX5_SET(general_obj_in_cmd_hdr, in, opcode,
181 MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
182 MLX5_SET(general_obj_in_cmd_hdr, in, obj_type,
183 MLX5_GENERAL_OBJECT_TYPES_IPSEC);
184
185 err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
186 if (!err)
187 *ipsec_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
188
189 return err;
190}
191
192static void mlx5_destroy_ipsec_obj(struct mlx5_core_dev *mdev, u32 ipsec_id)
193{
194 u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {};
195 u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
196
197 MLX5_SET(general_obj_in_cmd_hdr, in, opcode,
198 MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
199 MLX5_SET(general_obj_in_cmd_hdr, in, obj_type,
200 MLX5_GENERAL_OBJECT_TYPES_IPSEC);
201 MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, ipsec_id);
202
203 mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
204}
205
206static void *mlx5_ipsec_offload_create_sa_ctx(struct mlx5_core_dev *mdev,
207 struct mlx5_accel_esp_xfrm *accel_xfrm,
208 const __be32 saddr[4], const __be32 daddr[4],
209 const __be32 spi, bool is_ipv6, u32 *hw_handle)
210{
211 struct mlx5_accel_esp_xfrm_attrs *xfrm_attrs = &accel_xfrm->attrs;
212 struct aes_gcm_keymat *aes_gcm = &xfrm_attrs->keymat.aes_gcm;
213 struct mlx5_ipsec_obj_attrs ipsec_attrs = {};
214 struct mlx5_ipsec_esp_xfrm *mxfrm;
215 struct mlx5_ipsec_sa_ctx *sa_ctx;
216 int err;
217
218
219 sa_ctx = kzalloc(sizeof(*sa_ctx), GFP_KERNEL);
220 if (!sa_ctx)
221 return ERR_PTR(-ENOMEM);
222
223 sa_ctx->dev = mdev;
224
225 mxfrm = container_of(accel_xfrm, struct mlx5_ipsec_esp_xfrm, accel_xfrm);
226 mutex_lock(&mxfrm->lock);
227 sa_ctx->mxfrm = mxfrm;
228
229
230 err = mlx5_create_encryption_key(mdev, aes_gcm->aes_key,
231 aes_gcm->key_len / BITS_PER_BYTE,
232 MLX5_ACCEL_OBJ_IPSEC_KEY,
233 &sa_ctx->enc_key_id);
234 if (err) {
235 mlx5_core_dbg(mdev, "Failed to create encryption key (err = %d)\n", err);
236 goto err_sa_ctx;
237 }
238
239 ipsec_attrs.aes_gcm = aes_gcm;
240 ipsec_attrs.accel_flags = accel_xfrm->attrs.flags;
241 ipsec_attrs.esn_msb = accel_xfrm->attrs.esn;
242 ipsec_attrs.enc_key_id = sa_ctx->enc_key_id;
243 err = mlx5_create_ipsec_obj(mdev, &ipsec_attrs,
244 &sa_ctx->ipsec_obj_id);
245 if (err) {
246 mlx5_core_dbg(mdev, "Failed to create IPsec object (err = %d)\n", err);
247 goto err_enc_key;
248 }
249
250 *hw_handle = sa_ctx->ipsec_obj_id;
251 mxfrm->sa_ctx = sa_ctx;
252 mutex_unlock(&mxfrm->lock);
253
254 return sa_ctx;
255
256err_enc_key:
257 mlx5_destroy_encryption_key(mdev, sa_ctx->enc_key_id);
258err_sa_ctx:
259 mutex_unlock(&mxfrm->lock);
260 kfree(sa_ctx);
261 return ERR_PTR(err);
262}
263
264static void mlx5_ipsec_offload_delete_sa_ctx(void *context)
265{
266 struct mlx5_ipsec_sa_ctx *sa_ctx = (struct mlx5_ipsec_sa_ctx *)context;
267 struct mlx5_ipsec_esp_xfrm *mxfrm = sa_ctx->mxfrm;
268
269 mutex_lock(&mxfrm->lock);
270 mlx5_destroy_ipsec_obj(sa_ctx->dev, sa_ctx->ipsec_obj_id);
271 mlx5_destroy_encryption_key(sa_ctx->dev, sa_ctx->enc_key_id);
272 kfree(sa_ctx);
273 mxfrm->sa_ctx = NULL;
274 mutex_unlock(&mxfrm->lock);
275}
276
277static int mlx5_ipsec_offload_init(struct mlx5_core_dev *mdev)
278{
279 return 0;
280}
281
282static int mlx5_modify_ipsec_obj(struct mlx5_core_dev *mdev,
283 struct mlx5_ipsec_obj_attrs *attrs,
284 u32 ipsec_id)
285{
286 u32 in[MLX5_ST_SZ_DW(modify_ipsec_obj_in)] = {};
287 u32 out[MLX5_ST_SZ_DW(query_ipsec_obj_out)];
288 u64 modify_field_select = 0;
289 u64 general_obj_types;
290 void *obj;
291 int err;
292
293 if (!(attrs->accel_flags & MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED))
294 return 0;
295
296 general_obj_types = MLX5_CAP_GEN_64(mdev, general_obj_types);
297 if (!(general_obj_types & MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_IPSEC))
298 return -EINVAL;
299
300
301 MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_QUERY_GENERAL_OBJECT);
302 MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_IPSEC);
303 MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, ipsec_id);
304 err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
305 if (err) {
306 mlx5_core_err(mdev, "Query IPsec object failed (Object id %d), err = %d\n",
307 ipsec_id, err);
308 return err;
309 }
310
311 obj = MLX5_ADDR_OF(query_ipsec_obj_out, out, ipsec_object);
312 modify_field_select = MLX5_GET64(ipsec_obj, obj, modify_field_select);
313
314
315 if (!(modify_field_select & MLX5_MODIFY_IPSEC_BITMASK_ESN_OVERLAP) ||
316 !(modify_field_select & MLX5_MODIFY_IPSEC_BITMASK_ESN_MSB))
317 return -EOPNOTSUPP;
318
319 obj = MLX5_ADDR_OF(modify_ipsec_obj_in, in, ipsec_object);
320 MLX5_SET(ipsec_obj, obj, esn_msb, attrs->esn_msb);
321 if (attrs->accel_flags & MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP)
322 MLX5_SET(ipsec_obj, obj, esn_overlap, 1);
323
324
325 MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_MODIFY_GENERAL_OBJECT);
326
327 return mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
328}
329
330static int mlx5_ipsec_offload_esp_modify_xfrm(struct mlx5_accel_esp_xfrm *xfrm,
331 const struct mlx5_accel_esp_xfrm_attrs *attrs)
332{
333 struct mlx5_ipsec_obj_attrs ipsec_attrs = {};
334 struct mlx5_core_dev *mdev = xfrm->mdev;
335 struct mlx5_ipsec_esp_xfrm *mxfrm;
336
337 int err = 0;
338
339 if (!memcmp(&xfrm->attrs, attrs, sizeof(xfrm->attrs)))
340 return 0;
341
342 if (mlx5_ipsec_offload_esp_validate_xfrm_attrs(mdev, attrs))
343 return -EOPNOTSUPP;
344
345 mxfrm = container_of(xfrm, struct mlx5_ipsec_esp_xfrm, accel_xfrm);
346
347 mutex_lock(&mxfrm->lock);
348
349 if (!mxfrm->sa_ctx)
350
351 goto change_sw_xfrm_attrs;
352
353
354
355 ipsec_attrs.accel_flags = attrs->flags;
356 ipsec_attrs.esn_msb = attrs->esn;
357 err = mlx5_modify_ipsec_obj(mdev,
358 &ipsec_attrs,
359 mxfrm->sa_ctx->ipsec_obj_id);
360
361change_sw_xfrm_attrs:
362 if (!err)
363 memcpy(&xfrm->attrs, attrs, sizeof(xfrm->attrs));
364
365 mutex_unlock(&mxfrm->lock);
366 return err;
367}
368
369static const struct mlx5_accel_ipsec_ops ipsec_offload_ops = {
370 .device_caps = mlx5_ipsec_offload_device_caps,
371 .create_hw_context = mlx5_ipsec_offload_create_sa_ctx,
372 .free_hw_context = mlx5_ipsec_offload_delete_sa_ctx,
373 .init = mlx5_ipsec_offload_init,
374 .esp_create_xfrm = mlx5_ipsec_offload_esp_create_xfrm,
375 .esp_destroy_xfrm = mlx5_ipsec_offload_esp_destroy_xfrm,
376 .esp_modify_xfrm = mlx5_ipsec_offload_esp_modify_xfrm,
377};
378
379const struct mlx5_accel_ipsec_ops *mlx5_ipsec_offload_ops(struct mlx5_core_dev *mdev)
380{
381 if (!mlx5_ipsec_offload_device_caps(mdev))
382 return NULL;
383
384 return &ipsec_offload_ops;
385}
386
