1
2
3
4#include "en_accel/tls.h"
5#include "en_accel/ktls_txrx.h"
6#include "en_accel/ktls_utils.h"
7
8struct mlx5e_dump_wqe {
9 struct mlx5_wqe_ctrl_seg ctrl;
10 struct mlx5_wqe_data_seg data;
11};
12
13#define MLX5E_KTLS_DUMP_WQEBBS \
14 (DIV_ROUND_UP(sizeof(struct mlx5e_dump_wqe), MLX5_SEND_WQE_BB))
15
16static u8
17mlx5e_ktls_dumps_num_wqes(struct mlx5e_params *params, unsigned int nfrags,
18 unsigned int sync_len)
19{
20
21
22
23 return nfrags + DIV_ROUND_UP(sync_len, MLX5E_SW2HW_MTU(params, params->sw_mtu));
24}
25
26u16 mlx5e_ktls_get_stop_room(struct mlx5_core_dev *mdev, struct mlx5e_params *params)
27{
28 u16 num_dumps, stop_room = 0;
29
30 if (!mlx5e_accel_is_ktls_tx(mdev))
31 return 0;
32
33 num_dumps = mlx5e_ktls_dumps_num_wqes(params, MAX_SKB_FRAGS, TLS_MAX_PAYLOAD_SIZE);
34
35 stop_room += mlx5e_stop_room_for_wqe(MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS);
36 stop_room += mlx5e_stop_room_for_wqe(MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS);
37 stop_room += num_dumps * mlx5e_stop_room_for_wqe(MLX5E_KTLS_DUMP_WQEBBS);
38
39 return stop_room;
40}
41
42static int mlx5e_ktls_create_tis(struct mlx5_core_dev *mdev, u32 *tisn)
43{
44 u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {};
45 void *tisc;
46
47 tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
48
49 MLX5_SET(tisc, tisc, tls_en, 1);
50
51 return mlx5e_create_tis(mdev, in, tisn);
52}
53
54struct mlx5e_ktls_offload_context_tx {
55 struct tls_offload_context_tx *tx_ctx;
56 struct tls12_crypto_info_aes_gcm_128 crypto_info;
57 struct mlx5e_tls_sw_stats *sw_stats;
58 u32 expected_seq;
59 u32 tisn;
60 u32 key_id;
61 bool ctx_post_pending;
62};
63
64static void
65mlx5e_set_ktls_tx_priv_ctx(struct tls_context *tls_ctx,
66 struct mlx5e_ktls_offload_context_tx *priv_tx)
67{
68 struct mlx5e_ktls_offload_context_tx **ctx =
69 __tls_driver_ctx(tls_ctx, TLS_OFFLOAD_CTX_DIR_TX);
70
71 BUILD_BUG_ON(sizeof(struct mlx5e_ktls_offload_context_tx *) >
72 TLS_OFFLOAD_CONTEXT_SIZE_TX);
73
74 *ctx = priv_tx;
75}
76
77static struct mlx5e_ktls_offload_context_tx *
78mlx5e_get_ktls_tx_priv_ctx(struct tls_context *tls_ctx)
79{
80 struct mlx5e_ktls_offload_context_tx **ctx =
81 __tls_driver_ctx(tls_ctx, TLS_OFFLOAD_CTX_DIR_TX);
82
83 return *ctx;
84}
85
86int mlx5e_ktls_add_tx(struct net_device *netdev, struct sock *sk,
87 struct tls_crypto_info *crypto_info, u32 start_offload_tcp_sn)
88{
89 struct mlx5e_ktls_offload_context_tx *priv_tx;
90 struct tls_context *tls_ctx;
91 struct mlx5_core_dev *mdev;
92 struct mlx5e_priv *priv;
93 int err;
94
95 tls_ctx = tls_get_ctx(sk);
96 priv = netdev_priv(netdev);
97 mdev = priv->mdev;
98
99 priv_tx = kzalloc(sizeof(*priv_tx), GFP_KERNEL);
100 if (!priv_tx)
101 return -ENOMEM;
102
103 err = mlx5_ktls_create_key(mdev, crypto_info, &priv_tx->key_id);
104 if (err)
105 goto err_create_key;
106
107 priv_tx->sw_stats = &priv->tls->sw_stats;
108 priv_tx->expected_seq = start_offload_tcp_sn;
109 priv_tx->crypto_info =
110 *(struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
111 priv_tx->tx_ctx = tls_offload_ctx_tx(tls_ctx);
112
113 mlx5e_set_ktls_tx_priv_ctx(tls_ctx, priv_tx);
114
115 err = mlx5e_ktls_create_tis(mdev, &priv_tx->tisn);
116 if (err)
117 goto err_create_tis;
118
119 priv_tx->ctx_post_pending = true;
120 atomic64_inc(&priv_tx->sw_stats->tx_tls_ctx);
121
122 return 0;
123
124err_create_tis:
125 mlx5_ktls_destroy_key(mdev, priv_tx->key_id);
126err_create_key:
127 kfree(priv_tx);
128 return err;
129}
130
131void mlx5e_ktls_del_tx(struct net_device *netdev, struct tls_context *tls_ctx)
132{
133 struct mlx5e_ktls_offload_context_tx *priv_tx;
134 struct mlx5_core_dev *mdev;
135 struct mlx5e_priv *priv;
136
137 priv_tx = mlx5e_get_ktls_tx_priv_ctx(tls_ctx);
138 priv = netdev_priv(netdev);
139 mdev = priv->mdev;
140
141 atomic64_inc(&priv_tx->sw_stats->tx_tls_del);
142 mlx5e_destroy_tis(mdev, priv_tx->tisn);
143 mlx5_ktls_destroy_key(mdev, priv_tx->key_id);
144 kfree(priv_tx);
145}
146
147static void tx_fill_wi(struct mlx5e_txqsq *sq,
148 u16 pi, u8 num_wqebbs, u32 num_bytes,
149 struct page *page)
150{
151 struct mlx5e_tx_wqe_info *wi = &sq->db.wqe_info[pi];
152
153 *wi = (struct mlx5e_tx_wqe_info) {
154 .num_wqebbs = num_wqebbs,
155 .num_bytes = num_bytes,
156 .resync_dump_frag_page = page,
157 };
158}
159
160static bool
161mlx5e_ktls_tx_offload_test_and_clear_pending(struct mlx5e_ktls_offload_context_tx *priv_tx)
162{
163 bool ret = priv_tx->ctx_post_pending;
164
165 priv_tx->ctx_post_pending = false;
166
167 return ret;
168}
169
170static void
171post_static_params(struct mlx5e_txqsq *sq,
172 struct mlx5e_ktls_offload_context_tx *priv_tx,
173 bool fence)
174{
175 struct mlx5e_set_tls_static_params_wqe *wqe;
176 u16 pi, num_wqebbs;
177
178 num_wqebbs = MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS;
179 pi = mlx5e_txqsq_get_next_pi(sq, num_wqebbs);
180 wqe = MLX5E_TLS_FETCH_SET_STATIC_PARAMS_WQE(sq, pi);
181 mlx5e_ktls_build_static_params(wqe, sq->pc, sq->sqn, &priv_tx->crypto_info,
182 priv_tx->tisn, priv_tx->key_id, 0, fence,
183 TLS_OFFLOAD_CTX_DIR_TX);
184 tx_fill_wi(sq, pi, num_wqebbs, 0, NULL);
185 sq->pc += num_wqebbs;
186}
187
188static void
189post_progress_params(struct mlx5e_txqsq *sq,
190 struct mlx5e_ktls_offload_context_tx *priv_tx,
191 bool fence)
192{
193 struct mlx5e_set_tls_progress_params_wqe *wqe;
194 u16 pi, num_wqebbs;
195
196 num_wqebbs = MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS;
197 pi = mlx5e_txqsq_get_next_pi(sq, num_wqebbs);
198 wqe = MLX5E_TLS_FETCH_SET_PROGRESS_PARAMS_WQE(sq, pi);
199 mlx5e_ktls_build_progress_params(wqe, sq->pc, sq->sqn, priv_tx->tisn, fence, 0,
200 TLS_OFFLOAD_CTX_DIR_TX);
201 tx_fill_wi(sq, pi, num_wqebbs, 0, NULL);
202 sq->pc += num_wqebbs;
203}
204
205static void
206mlx5e_ktls_tx_post_param_wqes(struct mlx5e_txqsq *sq,
207 struct mlx5e_ktls_offload_context_tx *priv_tx,
208 bool skip_static_post, bool fence_first_post)
209{
210 bool progress_fence = skip_static_post || !fence_first_post;
211
212 if (!skip_static_post)
213 post_static_params(sq, priv_tx, fence_first_post);
214
215 post_progress_params(sq, priv_tx, progress_fence);
216}
217
218struct tx_sync_info {
219 u64 rcd_sn;
220 u32 sync_len;
221 int nr_frags;
222 skb_frag_t frags[MAX_SKB_FRAGS];
223};
224
225enum mlx5e_ktls_sync_retval {
226 MLX5E_KTLS_SYNC_DONE,
227 MLX5E_KTLS_SYNC_FAIL,
228 MLX5E_KTLS_SYNC_SKIP_NO_DATA,
229};
230
231static enum mlx5e_ktls_sync_retval
232tx_sync_info_get(struct mlx5e_ktls_offload_context_tx *priv_tx,
233 u32 tcp_seq, int datalen, struct tx_sync_info *info)
234{
235 struct tls_offload_context_tx *tx_ctx = priv_tx->tx_ctx;
236 enum mlx5e_ktls_sync_retval ret = MLX5E_KTLS_SYNC_DONE;
237 struct tls_record_info *record;
238 int remaining, i = 0;
239 unsigned long flags;
240 bool ends_before;
241
242 spin_lock_irqsave(&tx_ctx->lock, flags);
243 record = tls_get_record(tx_ctx, tcp_seq, &info->rcd_sn);
244
245 if (unlikely(!record)) {
246 ret = MLX5E_KTLS_SYNC_FAIL;
247 goto out;
248 }
249
250
251
252
253
254
255
256
257
258 ends_before = before(tcp_seq + datalen - 1, tls_record_start_seq(record));
259
260 if (unlikely(tls_record_is_start_marker(record))) {
261 ret = ends_before ? MLX5E_KTLS_SYNC_SKIP_NO_DATA : MLX5E_KTLS_SYNC_FAIL;
262 goto out;
263 } else if (ends_before) {
264 ret = MLX5E_KTLS_SYNC_FAIL;
265 goto out;
266 }
267
268 info->sync_len = tcp_seq - tls_record_start_seq(record);
269 remaining = info->sync_len;
270 while (remaining > 0) {
271 skb_frag_t *frag = &record->frags[i];
272
273 get_page(skb_frag_page(frag));
274 remaining -= skb_frag_size(frag);
275 info->frags[i++] = *frag;
276 }
277
278 if (remaining < 0)
279 skb_frag_size_add(&info->frags[i - 1], remaining);
280 info->nr_frags = i;
281out:
282 spin_unlock_irqrestore(&tx_ctx->lock, flags);
283 return ret;
284}
285
286static void
287tx_post_resync_params(struct mlx5e_txqsq *sq,
288 struct mlx5e_ktls_offload_context_tx *priv_tx,
289 u64 rcd_sn)
290{
291 struct tls12_crypto_info_aes_gcm_128 *info = &priv_tx->crypto_info;
292 __be64 rn_be = cpu_to_be64(rcd_sn);
293 bool skip_static_post;
294 u16 rec_seq_sz;
295 char *rec_seq;
296
297 rec_seq = info->rec_seq;
298 rec_seq_sz = sizeof(info->rec_seq);
299
300 skip_static_post = !memcmp(rec_seq, &rn_be, rec_seq_sz);
301 if (!skip_static_post)
302 memcpy(rec_seq, &rn_be, rec_seq_sz);
303
304 mlx5e_ktls_tx_post_param_wqes(sq, priv_tx, skip_static_post, true);
305}
306
307static int
308tx_post_resync_dump(struct mlx5e_txqsq *sq, skb_frag_t *frag, u32 tisn, bool first)
309{
310 struct mlx5_wqe_ctrl_seg *cseg;
311 struct mlx5_wqe_data_seg *dseg;
312 struct mlx5e_dump_wqe *wqe;
313 dma_addr_t dma_addr = 0;
314 u16 ds_cnt;
315 int fsz;
316 u16 pi;
317
318 BUILD_BUG_ON(MLX5E_KTLS_DUMP_WQEBBS != 1);
319 pi = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->pc);
320 wqe = MLX5E_TLS_FETCH_DUMP_WQE(sq, pi);
321
322 ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS;
323
324 cseg = &wqe->ctrl;
325 dseg = &wqe->data;
326
327 cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_DUMP);
328 cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
329 cseg->tis_tir_num = cpu_to_be32(tisn << 8);
330 cseg->fm_ce_se = first ? MLX5_FENCE_MODE_INITIATOR_SMALL : 0;
331
332 fsz = skb_frag_size(frag);
333 dma_addr = skb_frag_dma_map(sq->pdev, frag, 0, fsz,
334 DMA_TO_DEVICE);
335 if (unlikely(dma_mapping_error(sq->pdev, dma_addr)))
336 return -ENOMEM;
337
338 dseg->addr = cpu_to_be64(dma_addr);
339 dseg->lkey = sq->mkey_be;
340 dseg->byte_count = cpu_to_be32(fsz);
341 mlx5e_dma_push(sq, dma_addr, fsz, MLX5E_DMA_MAP_PAGE);
342
343 tx_fill_wi(sq, pi, MLX5E_KTLS_DUMP_WQEBBS, fsz, skb_frag_page(frag));
344 sq->pc += MLX5E_KTLS_DUMP_WQEBBS;
345
346 return 0;
347}
348
349void mlx5e_ktls_tx_handle_resync_dump_comp(struct mlx5e_txqsq *sq,
350 struct mlx5e_tx_wqe_info *wi,
351 u32 *dma_fifo_cc)
352{
353 struct mlx5e_sq_stats *stats;
354 struct mlx5e_sq_dma *dma;
355
356 dma = mlx5e_dma_get(sq, (*dma_fifo_cc)++);
357 stats = sq->stats;
358
359 mlx5e_tx_dma_unmap(sq->pdev, dma);
360 put_page(wi->resync_dump_frag_page);
361 stats->tls_dump_packets++;
362 stats->tls_dump_bytes += wi->num_bytes;
363}
364
365static void tx_post_fence_nop(struct mlx5e_txqsq *sq)
366{
367 struct mlx5_wq_cyc *wq = &sq->wq;
368 u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
369
370 tx_fill_wi(sq, pi, 1, 0, NULL);
371
372 mlx5e_post_nop_fence(wq, sq->sqn, &sq->pc);
373}
374
375static enum mlx5e_ktls_sync_retval
376mlx5e_ktls_tx_handle_ooo(struct mlx5e_ktls_offload_context_tx *priv_tx,
377 struct mlx5e_txqsq *sq,
378 int datalen,
379 u32 seq)
380{
381 struct mlx5e_sq_stats *stats = sq->stats;
382 enum mlx5e_ktls_sync_retval ret;
383 struct tx_sync_info info = {};
384 int i = 0;
385
386 ret = tx_sync_info_get(priv_tx, seq, datalen, &info);
387 if (unlikely(ret != MLX5E_KTLS_SYNC_DONE)) {
388 if (ret == MLX5E_KTLS_SYNC_SKIP_NO_DATA) {
389 stats->tls_skip_no_sync_data++;
390 return MLX5E_KTLS_SYNC_SKIP_NO_DATA;
391 }
392
393
394
395
396 stats->tls_drop_no_sync_data++;
397 goto err_out;
398 }
399
400 stats->tls_ooo++;
401
402 tx_post_resync_params(sq, priv_tx, info.rcd_sn);
403
404
405
406
407 if (!info.nr_frags) {
408 tx_post_fence_nop(sq);
409 return MLX5E_KTLS_SYNC_DONE;
410 }
411
412 for (; i < info.nr_frags; i++) {
413 unsigned int orig_fsz, frag_offset = 0, n = 0;
414 skb_frag_t *f = &info.frags[i];
415
416 orig_fsz = skb_frag_size(f);
417
418 do {
419 bool fence = !(i || frag_offset);
420 unsigned int fsz;
421
422 n++;
423 fsz = min_t(unsigned int, sq->hw_mtu, orig_fsz - frag_offset);
424 skb_frag_size_set(f, fsz);
425 if (tx_post_resync_dump(sq, f, priv_tx->tisn, fence)) {
426 page_ref_add(skb_frag_page(f), n - 1);
427 goto err_out;
428 }
429
430 skb_frag_off_add(f, fsz);
431 frag_offset += fsz;
432 } while (frag_offset < orig_fsz);
433
434 page_ref_add(skb_frag_page(f), n - 1);
435 }
436
437 return MLX5E_KTLS_SYNC_DONE;
438
439err_out:
440 for (; i < info.nr_frags; i++)
441
442
443
444
445
446 put_page(skb_frag_page(&info.frags[i]));
447
448 return MLX5E_KTLS_SYNC_FAIL;
449}
450
451bool mlx5e_ktls_handle_tx_skb(struct tls_context *tls_ctx, struct mlx5e_txqsq *sq,
452 struct sk_buff *skb, int datalen,
453 struct mlx5e_accel_tx_tls_state *state)
454{
455 struct mlx5e_ktls_offload_context_tx *priv_tx;
456 struct mlx5e_sq_stats *stats = sq->stats;
457 u32 seq;
458
459 priv_tx = mlx5e_get_ktls_tx_priv_ctx(tls_ctx);
460
461 if (unlikely(mlx5e_ktls_tx_offload_test_and_clear_pending(priv_tx))) {
462 mlx5e_ktls_tx_post_param_wqes(sq, priv_tx, false, false);
463 }
464
465 seq = ntohl(tcp_hdr(skb)->seq);
466 if (unlikely(priv_tx->expected_seq != seq)) {
467 enum mlx5e_ktls_sync_retval ret =
468 mlx5e_ktls_tx_handle_ooo(priv_tx, sq, datalen, seq);
469
470 switch (ret) {
471 case MLX5E_KTLS_SYNC_DONE:
472 break;
473 case MLX5E_KTLS_SYNC_SKIP_NO_DATA:
474 if (likely(!skb->decrypted))
475 goto out;
476 WARN_ON_ONCE(1);
477 fallthrough;
478 case MLX5E_KTLS_SYNC_FAIL:
479 goto err_out;
480 }
481 }
482
483 priv_tx->expected_seq = seq + datalen;
484
485 state->tls_tisn = priv_tx->tisn;
486
487 stats->tls_encrypted_packets += skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1;
488 stats->tls_encrypted_bytes += datalen;
489
490out:
491 return true;
492
493err_out:
494 dev_kfree_skb_any(skb);
495 return false;
496}
497