1
2
3
4#include <linux/bitfield.h>
5#include <linux/ipv6.h>
6#include <linux/skbuff.h>
7#include <linux/string.h>
8#include <net/tls.h>
9
10#include "../ccm.h"
11#include "../nfp_net.h"
12#include "crypto.h"
13#include "fw.h"
14
15#define NFP_NET_TLS_CCM_MBOX_OPS_MASK \
16 (BIT(NFP_CCM_TYPE_CRYPTO_RESET) | \
17 BIT(NFP_CCM_TYPE_CRYPTO_ADD) | \
18 BIT(NFP_CCM_TYPE_CRYPTO_DEL) | \
19 BIT(NFP_CCM_TYPE_CRYPTO_UPDATE))
20
21#define NFP_NET_TLS_OPCODE_MASK_RX \
22 BIT(NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_DEC)
23
24#define NFP_NET_TLS_OPCODE_MASK_TX \
25 BIT(NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_ENC)
26
27#define NFP_NET_TLS_OPCODE_MASK \
28 (NFP_NET_TLS_OPCODE_MASK_RX | NFP_NET_TLS_OPCODE_MASK_TX)
29
30static void nfp_net_crypto_set_op(struct nfp_net *nn, u8 opcode, bool on)
31{
32 u32 off, val;
33
34 off = nn->tlv_caps.crypto_enable_off + round_down(opcode / 8, 4);
35
36 val = nn_readl(nn, off);
37 if (on)
38 val |= BIT(opcode & 31);
39 else
40 val &= ~BIT(opcode & 31);
41 nn_writel(nn, off, val);
42}
43
44static bool
45__nfp_net_tls_conn_cnt_changed(struct nfp_net *nn, int add,
46 enum tls_offload_ctx_dir direction)
47{
48 u8 opcode;
49 int cnt;
50
51 if (direction == TLS_OFFLOAD_CTX_DIR_TX) {
52 opcode = NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_ENC;
53 nn->ktls_tx_conn_cnt += add;
54 cnt = nn->ktls_tx_conn_cnt;
55 nn->dp.ktls_tx = !!nn->ktls_tx_conn_cnt;
56 } else {
57 opcode = NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_DEC;
58 nn->ktls_rx_conn_cnt += add;
59 cnt = nn->ktls_rx_conn_cnt;
60 }
61
62
63 if (cnt > 1)
64 return false;
65
66 nfp_net_crypto_set_op(nn, opcode, cnt);
67 return true;
68}
69
70static int
71nfp_net_tls_conn_cnt_changed(struct nfp_net *nn, int add,
72 enum tls_offload_ctx_dir direction)
73{
74 int ret = 0;
75
76
77 nn_ctrl_bar_lock(nn);
78 if (__nfp_net_tls_conn_cnt_changed(nn, add, direction)) {
79 ret = __nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_CRYPTO);
80
81 if (ret)
82 __nfp_net_tls_conn_cnt_changed(nn, -add, direction);
83 }
84 nn_ctrl_bar_unlock(nn);
85
86 return ret;
87}
88
89static int
90nfp_net_tls_conn_add(struct nfp_net *nn, enum tls_offload_ctx_dir direction)
91{
92 return nfp_net_tls_conn_cnt_changed(nn, 1, direction);
93}
94
95static int
96nfp_net_tls_conn_remove(struct nfp_net *nn, enum tls_offload_ctx_dir direction)
97{
98 return nfp_net_tls_conn_cnt_changed(nn, -1, direction);
99}
100
101static struct sk_buff *
102nfp_net_tls_alloc_simple(struct nfp_net *nn, size_t req_sz, gfp_t flags)
103{
104 return nfp_ccm_mbox_msg_alloc(nn, req_sz,
105 sizeof(struct nfp_crypto_reply_simple),
106 flags);
107}
108
109static int
110nfp_net_tls_communicate_simple(struct nfp_net *nn, struct sk_buff *skb,
111 const char *name, enum nfp_ccm_type type)
112{
113 struct nfp_crypto_reply_simple *reply;
114 int err;
115
116 err = __nfp_ccm_mbox_communicate(nn, skb, type,
117 sizeof(*reply), sizeof(*reply),
118 type == NFP_CCM_TYPE_CRYPTO_DEL);
119 if (err) {
120 nn_dp_warn(&nn->dp, "failed to %s TLS: %d\n", name, err);
121 return err;
122 }
123
124 reply = (void *)skb->data;
125 err = -be32_to_cpu(reply->error);
126 if (err)
127 nn_dp_warn(&nn->dp, "failed to %s TLS, fw replied: %d\n",
128 name, err);
129 dev_consume_skb_any(skb);
130
131 return err;
132}
133
134static void nfp_net_tls_del_fw(struct nfp_net *nn, __be32 *fw_handle)
135{
136 struct nfp_crypto_req_del *req;
137 struct sk_buff *skb;
138
139 skb = nfp_net_tls_alloc_simple(nn, sizeof(*req), GFP_KERNEL);
140 if (!skb)
141 return;
142
143 req = (void *)skb->data;
144 req->ep_id = 0;
145 memcpy(req->handle, fw_handle, sizeof(req->handle));
146
147 nfp_net_tls_communicate_simple(nn, skb, "delete",
148 NFP_CCM_TYPE_CRYPTO_DEL);
149}
150
151static void
152nfp_net_tls_set_ipver_vlan(struct nfp_crypto_req_add_front *front, u8 ipver)
153{
154 front->ipver_vlan = cpu_to_be16(FIELD_PREP(NFP_NET_TLS_IPVER, ipver) |
155 FIELD_PREP(NFP_NET_TLS_VLAN,
156 NFP_NET_TLS_VLAN_UNUSED));
157}
158
159static void
160nfp_net_tls_assign_conn_id(struct nfp_net *nn,
161 struct nfp_crypto_req_add_front *front)
162{
163 u32 len;
164 u64 id;
165
166 id = atomic64_inc_return(&nn->ktls_conn_id_gen);
167 len = front->key_len - NFP_NET_TLS_NON_ADDR_KEY_LEN;
168
169 memcpy(front->l3_addrs, &id, sizeof(id));
170 memset(front->l3_addrs + sizeof(id), 0, len - sizeof(id));
171}
172
173static struct nfp_crypto_req_add_back *
174nfp_net_tls_set_ipv4(struct nfp_net *nn, struct nfp_crypto_req_add_v4 *req,
175 struct sock *sk, int direction)
176{
177 struct inet_sock *inet = inet_sk(sk);
178
179 req->front.key_len += sizeof(__be32) * 2;
180
181 if (direction == TLS_OFFLOAD_CTX_DIR_TX) {
182 nfp_net_tls_assign_conn_id(nn, &req->front);
183 } else {
184 req->src_ip = inet->inet_daddr;
185 req->dst_ip = inet->inet_saddr;
186 }
187
188 return &req->back;
189}
190
191static struct nfp_crypto_req_add_back *
192nfp_net_tls_set_ipv6(struct nfp_net *nn, struct nfp_crypto_req_add_v6 *req,
193 struct sock *sk, int direction)
194{
195#if IS_ENABLED(CONFIG_IPV6)
196 struct ipv6_pinfo *np = inet6_sk(sk);
197
198 req->front.key_len += sizeof(struct in6_addr) * 2;
199
200 if (direction == TLS_OFFLOAD_CTX_DIR_TX) {
201 nfp_net_tls_assign_conn_id(nn, &req->front);
202 } else {
203 memcpy(req->src_ip, &sk->sk_v6_daddr, sizeof(req->src_ip));
204 memcpy(req->dst_ip, &np->saddr, sizeof(req->dst_ip));
205 }
206
207#endif
208 return &req->back;
209}
210
211static void
212nfp_net_tls_set_l4(struct nfp_crypto_req_add_front *front,
213 struct nfp_crypto_req_add_back *back, struct sock *sk,
214 int direction)
215{
216 struct inet_sock *inet = inet_sk(sk);
217
218 front->l4_proto = IPPROTO_TCP;
219
220 if (direction == TLS_OFFLOAD_CTX_DIR_TX) {
221 back->src_port = 0;
222 back->dst_port = 0;
223 } else {
224 back->src_port = inet->inet_dport;
225 back->dst_port = inet->inet_sport;
226 }
227}
228
229static u8 nfp_tls_1_2_dir_to_opcode(enum tls_offload_ctx_dir direction)
230{
231 switch (direction) {
232 case TLS_OFFLOAD_CTX_DIR_TX:
233 return NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_ENC;
234 case TLS_OFFLOAD_CTX_DIR_RX:
235 return NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_DEC;
236 default:
237 WARN_ON_ONCE(1);
238 return 0;
239 }
240}
241
242static bool
243nfp_net_cipher_supported(struct nfp_net *nn, u16 cipher_type,
244 enum tls_offload_ctx_dir direction)
245{
246 u8 bit;
247
248 switch (cipher_type) {
249 case TLS_CIPHER_AES_GCM_128:
250 if (direction == TLS_OFFLOAD_CTX_DIR_TX)
251 bit = NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_ENC;
252 else
253 bit = NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_DEC;
254 break;
255 default:
256 return false;
257 }
258
259 return nn->tlv_caps.crypto_ops & BIT(bit);
260}
261
262static int
263nfp_net_tls_add(struct net_device *netdev, struct sock *sk,
264 enum tls_offload_ctx_dir direction,
265 struct tls_crypto_info *crypto_info,
266 u32 start_offload_tcp_sn)
267{
268 struct tls12_crypto_info_aes_gcm_128 *tls_ci;
269 struct nfp_net *nn = netdev_priv(netdev);
270 struct nfp_crypto_req_add_front *front;
271 struct nfp_net_tls_offload_ctx *ntls;
272 struct nfp_crypto_req_add_back *back;
273 struct nfp_crypto_reply_add *reply;
274 struct sk_buff *skb;
275 size_t req_sz;
276 void *req;
277 bool ipv6;
278 int err;
279
280 BUILD_BUG_ON(sizeof(struct nfp_net_tls_offload_ctx) >
281 TLS_DRIVER_STATE_SIZE_TX);
282 BUILD_BUG_ON(offsetof(struct nfp_net_tls_offload_ctx, rx_end) >
283 TLS_DRIVER_STATE_SIZE_RX);
284
285 if (!nfp_net_cipher_supported(nn, crypto_info->cipher_type, direction))
286 return -EOPNOTSUPP;
287
288 switch (sk->sk_family) {
289#if IS_ENABLED(CONFIG_IPV6)
290 case AF_INET6:
291 if (sk->sk_ipv6only ||
292 ipv6_addr_type(&sk->sk_v6_daddr) != IPV6_ADDR_MAPPED) {
293 req_sz = sizeof(struct nfp_crypto_req_add_v6);
294 ipv6 = true;
295 break;
296 }
297#endif
298
299 case AF_INET:
300 req_sz = sizeof(struct nfp_crypto_req_add_v4);
301 ipv6 = false;
302 break;
303 default:
304 return -EOPNOTSUPP;
305 }
306
307 err = nfp_net_tls_conn_add(nn, direction);
308 if (err)
309 return err;
310
311 skb = nfp_ccm_mbox_msg_alloc(nn, req_sz, sizeof(*reply), GFP_KERNEL);
312 if (!skb) {
313 err = -ENOMEM;
314 goto err_conn_remove;
315 }
316
317 front = (void *)skb->data;
318 front->ep_id = 0;
319 front->key_len = NFP_NET_TLS_NON_ADDR_KEY_LEN;
320 front->opcode = nfp_tls_1_2_dir_to_opcode(direction);
321 memset(front->resv, 0, sizeof(front->resv));
322
323 nfp_net_tls_set_ipver_vlan(front, ipv6 ? 6 : 4);
324
325 req = (void *)skb->data;
326 if (ipv6)
327 back = nfp_net_tls_set_ipv6(nn, req, sk, direction);
328 else
329 back = nfp_net_tls_set_ipv4(nn, req, sk, direction);
330
331 nfp_net_tls_set_l4(front, back, sk, direction);
332
333 back->counter = 0;
334 back->tcp_seq = cpu_to_be32(start_offload_tcp_sn);
335
336 tls_ci = (struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
337 memcpy(back->key, tls_ci->key, TLS_CIPHER_AES_GCM_128_KEY_SIZE);
338 memset(&back->key[TLS_CIPHER_AES_GCM_128_KEY_SIZE / 4], 0,
339 sizeof(back->key) - TLS_CIPHER_AES_GCM_128_KEY_SIZE);
340 memcpy(back->iv, tls_ci->iv, TLS_CIPHER_AES_GCM_128_IV_SIZE);
341 memcpy(&back->salt, tls_ci->salt, TLS_CIPHER_AES_GCM_128_SALT_SIZE);
342 memcpy(back->rec_no, tls_ci->rec_seq, sizeof(tls_ci->rec_seq));
343
344
345 skb_get(skb);
346
347 err = nfp_ccm_mbox_communicate(nn, skb, NFP_CCM_TYPE_CRYPTO_ADD,
348 sizeof(*reply), sizeof(*reply));
349 reply = (void *)skb->data;
350
351
352
353
354 if (!WARN_ON_ONCE((u8 *)back < skb->head ||
355 (u8 *)back > skb_end_pointer(skb)) &&
356 !WARN_ON_ONCE((u8 *)&reply[1] > (u8 *)back))
357 memzero_explicit(back, sizeof(*back));
358 dev_consume_skb_any(skb);
359
360 if (err) {
361 nn_dp_warn(&nn->dp, "failed to add TLS: %d (%d)\n",
362 err, direction == TLS_OFFLOAD_CTX_DIR_TX);
363
364 goto err_conn_remove;
365 }
366
367 err = -be32_to_cpu(reply->error);
368 if (err) {
369 if (err == -ENOSPC) {
370 if (!atomic_fetch_inc(&nn->ktls_no_space))
371 nn_info(nn, "HW TLS table full\n");
372 } else {
373 nn_dp_warn(&nn->dp,
374 "failed to add TLS, FW replied: %d\n", err);
375 }
376 goto err_free_skb;
377 }
378
379 if (!reply->handle[0] && !reply->handle[1]) {
380 nn_dp_warn(&nn->dp, "FW returned NULL handle\n");
381 err = -EINVAL;
382 goto err_fw_remove;
383 }
384
385 ntls = tls_driver_ctx(sk, direction);
386 memcpy(ntls->fw_handle, reply->handle, sizeof(ntls->fw_handle));
387 if (direction == TLS_OFFLOAD_CTX_DIR_TX)
388 ntls->next_seq = start_offload_tcp_sn;
389 dev_consume_skb_any(skb);
390
391 if (direction == TLS_OFFLOAD_CTX_DIR_TX)
392 return 0;
393
394 tls_offload_rx_resync_set_type(sk,
395 TLS_OFFLOAD_SYNC_TYPE_CORE_NEXT_HINT);
396 return 0;
397
398err_fw_remove:
399 nfp_net_tls_del_fw(nn, reply->handle);
400err_free_skb:
401 dev_consume_skb_any(skb);
402err_conn_remove:
403 nfp_net_tls_conn_remove(nn, direction);
404 return err;
405}
406
407static void
408nfp_net_tls_del(struct net_device *netdev, struct tls_context *tls_ctx,
409 enum tls_offload_ctx_dir direction)
410{
411 struct nfp_net *nn = netdev_priv(netdev);
412 struct nfp_net_tls_offload_ctx *ntls;
413
414 nfp_net_tls_conn_remove(nn, direction);
415
416 ntls = __tls_driver_ctx(tls_ctx, direction);
417 nfp_net_tls_del_fw(nn, ntls->fw_handle);
418}
419
420static int
421nfp_net_tls_resync(struct net_device *netdev, struct sock *sk, u32 seq,
422 u8 *rcd_sn, enum tls_offload_ctx_dir direction)
423{
424 struct nfp_net *nn = netdev_priv(netdev);
425 struct nfp_net_tls_offload_ctx *ntls;
426 struct nfp_crypto_req_update *req;
427 struct sk_buff *skb;
428 gfp_t flags;
429 int err;
430
431 flags = direction == TLS_OFFLOAD_CTX_DIR_TX ? GFP_KERNEL : GFP_ATOMIC;
432 skb = nfp_net_tls_alloc_simple(nn, sizeof(*req), flags);
433 if (!skb)
434 return -ENOMEM;
435
436 ntls = tls_driver_ctx(sk, direction);
437 req = (void *)skb->data;
438 req->ep_id = 0;
439 req->opcode = nfp_tls_1_2_dir_to_opcode(direction);
440 memset(req->resv, 0, sizeof(req->resv));
441 memcpy(req->handle, ntls->fw_handle, sizeof(ntls->fw_handle));
442 req->tcp_seq = cpu_to_be32(seq);
443 memcpy(req->rec_no, rcd_sn, sizeof(req->rec_no));
444
445 if (direction == TLS_OFFLOAD_CTX_DIR_TX) {
446 err = nfp_net_tls_communicate_simple(nn, skb, "sync",
447 NFP_CCM_TYPE_CRYPTO_UPDATE);
448 if (err)
449 return err;
450 ntls->next_seq = seq;
451 } else {
452 nfp_ccm_mbox_post(nn, skb, NFP_CCM_TYPE_CRYPTO_UPDATE,
453 sizeof(struct nfp_crypto_reply_simple));
454 }
455
456 return 0;
457}
458
459static const struct tlsdev_ops nfp_net_tls_ops = {
460 .tls_dev_add = nfp_net_tls_add,
461 .tls_dev_del = nfp_net_tls_del,
462 .tls_dev_resync = nfp_net_tls_resync,
463};
464
465static int nfp_net_tls_reset(struct nfp_net *nn)
466{
467 struct nfp_crypto_req_reset *req;
468 struct sk_buff *skb;
469
470 skb = nfp_net_tls_alloc_simple(nn, sizeof(*req), GFP_KERNEL);
471 if (!skb)
472 return -ENOMEM;
473
474 req = (void *)skb->data;
475 req->ep_id = 0;
476
477 return nfp_net_tls_communicate_simple(nn, skb, "reset",
478 NFP_CCM_TYPE_CRYPTO_RESET);
479}
480
481int nfp_net_tls_init(struct nfp_net *nn)
482{
483 struct net_device *netdev = nn->dp.netdev;
484 int err;
485
486 if (!(nn->tlv_caps.crypto_ops & NFP_NET_TLS_OPCODE_MASK))
487 return 0;
488
489 if ((nn->tlv_caps.mbox_cmsg_types & NFP_NET_TLS_CCM_MBOX_OPS_MASK) !=
490 NFP_NET_TLS_CCM_MBOX_OPS_MASK)
491 return 0;
492
493 if (!nfp_ccm_mbox_fits(nn, sizeof(struct nfp_crypto_req_add_v6))) {
494 nn_warn(nn, "disabling TLS offload - mbox too small: %d\n",
495 nn->tlv_caps.mbox_len);
496 return 0;
497 }
498
499 err = nfp_net_tls_reset(nn);
500 if (err)
501 return err;
502
503 nn_ctrl_bar_lock(nn);
504 nn_writel(nn, nn->tlv_caps.crypto_enable_off, 0);
505 err = __nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_CRYPTO);
506 nn_ctrl_bar_unlock(nn);
507 if (err)
508 return err;
509
510 if (nn->tlv_caps.crypto_ops & NFP_NET_TLS_OPCODE_MASK_RX) {
511 netdev->hw_features |= NETIF_F_HW_TLS_RX;
512 netdev->features |= NETIF_F_HW_TLS_RX;
513 }
514 if (nn->tlv_caps.crypto_ops & NFP_NET_TLS_OPCODE_MASK_TX) {
515 netdev->hw_features |= NETIF_F_HW_TLS_TX;
516 netdev->features |= NETIF_F_HW_TLS_TX;
517 }
518
519 netdev->tlsdev_ops = &nfp_net_tls_ops;
520
521 return 0;
522}
523
