1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34#ifndef _TLS_OFFLOAD_H
35#define _TLS_OFFLOAD_H
36
37#include <linux/types.h>
38#include <asm/byteorder.h>
39#include <linux/crypto.h>
40#include <linux/socket.h>
41#include <linux/tcp.h>
42#include <linux/skmsg.h>
43#include <linux/mutex.h>
44#include <linux/netdevice.h>
45#include <linux/rcupdate.h>
46
47#include <net/net_namespace.h>
48#include <net/tcp.h>
49#include <net/strparser.h>
50#include <crypto/aead.h>
51#include <uapi/linux/tls.h>
52
53
54
55#define TLS_MAX_PAYLOAD_SIZE ((size_t)1 << 14)
56
57#define TLS_HEADER_SIZE 5
58#define TLS_NONCE_OFFSET TLS_HEADER_SIZE
59
60#define TLS_CRYPTO_INFO_READY(info) ((info)->cipher_type)
61
62#define TLS_RECORD_TYPE_DATA 0x17
63
64#define TLS_AAD_SPACE_SIZE 13
65
66#define MAX_IV_SIZE 16
67#define TLS_MAX_REC_SEQ_SIZE 8
68
69
70
71
72
73
74
75
76#define TLS_AES_CCM_IV_B0_BYTE 2
77
78#define __TLS_INC_STATS(net, field) \
79 __SNMP_INC_STATS((net)->mib.tls_statistics, field)
80#define TLS_INC_STATS(net, field) \
81 SNMP_INC_STATS((net)->mib.tls_statistics, field)
82#define __TLS_DEC_STATS(net, field) \
83 __SNMP_DEC_STATS((net)->mib.tls_statistics, field)
84#define TLS_DEC_STATS(net, field) \
85 SNMP_DEC_STATS((net)->mib.tls_statistics, field)
86
87enum {
88 TLS_BASE,
89 TLS_SW,
90 TLS_HW,
91 TLS_HW_RECORD,
92 TLS_NUM_CONFIG,
93};
94
95
96
97
98
99struct tls_rec {
100 struct list_head list;
101 int tx_ready;
102 int tx_flags;
103
104 struct sk_msg msg_plaintext;
105 struct sk_msg msg_encrypted;
106
107
108 struct scatterlist sg_aead_in[2];
109
110 struct scatterlist sg_aead_out[2];
111
112 char content_type;
113 struct scatterlist sg_content_type;
114
115 char aad_space[TLS_AAD_SPACE_SIZE];
116 u8 iv_data[MAX_IV_SIZE];
117 struct aead_request aead_req;
118 u8 aead_req_ctx[];
119};
120
121struct tls_msg {
122 struct strp_msg rxm;
123 u8 control;
124};
125
126struct tx_work {
127 struct delayed_work work;
128 struct sock *sk;
129};
130
131struct tls_sw_context_tx {
132 struct crypto_aead *aead_send;
133 struct crypto_wait async_wait;
134 struct tx_work tx_work;
135 struct tls_rec *open_rec;
136 struct list_head tx_list;
137 atomic_t encrypt_pending;
138 int async_notify;
139 u8 async_capable:1;
140
141#define BIT_TX_SCHEDULED 0
142#define BIT_TX_CLOSING 1
143 unsigned long tx_bitmask;
144};
145
146struct tls_sw_context_rx {
147 struct crypto_aead *aead_recv;
148 struct crypto_wait async_wait;
149 struct strparser strp;
150 struct sk_buff_head rx_list;
151 void (*saved_data_ready)(struct sock *sk);
152
153 struct sk_buff *recv_pkt;
154 u8 control;
155 u8 async_capable:1;
156 u8 decrypted:1;
157 atomic_t decrypt_pending;
158 bool async_notify;
159};
160
161struct tls_record_info {
162 struct list_head list;
163 u32 end_seq;
164 int len;
165 int num_frags;
166 skb_frag_t frags[MAX_SKB_FRAGS];
167};
168
169struct tls_offload_context_tx {
170 struct crypto_aead *aead_send;
171 spinlock_t lock;
172 struct list_head records_list;
173 struct tls_record_info *open_record;
174 struct tls_record_info *retransmit_hint;
175 u64 hint_record_sn;
176 u64 unacked_record_sn;
177
178 struct scatterlist sg_tx_data[MAX_SKB_FRAGS];
179 void (*sk_destruct)(struct sock *sk);
180 u8 driver_state[] __aligned(8);
181
182
183
184
185#define TLS_DRIVER_STATE_SIZE_TX 16
186};
187
188#define TLS_OFFLOAD_CONTEXT_SIZE_TX \
189 (sizeof(struct tls_offload_context_tx) + TLS_DRIVER_STATE_SIZE_TX)
190
191enum tls_context_flags {
192 TLS_RX_SYNC_RUNNING = 0,
193
194
195
196
197 TLS_TX_SYNC_SCHED = 1,
198};
199
200struct cipher_context {
201 char *iv;
202 char *rec_seq;
203};
204
205union tls_crypto_context {
206 struct tls_crypto_info info;
207 union {
208 struct tls12_crypto_info_aes_gcm_128 aes_gcm_128;
209 struct tls12_crypto_info_aes_gcm_256 aes_gcm_256;
210 };
211};
212
213struct tls_prot_info {
214 u16 version;
215 u16 cipher_type;
216 u16 prepend_size;
217 u16 tag_size;
218 u16 overhead_size;
219 u16 iv_size;
220 u16 salt_size;
221 u16 rec_seq_size;
222 u16 aad_size;
223 u16 tail_size;
224};
225
226struct tls_context {
227
228 struct tls_prot_info prot_info;
229
230 u8 tx_conf:3;
231 u8 rx_conf:3;
232
233 int (*push_pending_record)(struct sock *sk, int flags);
234 void (*sk_write_space)(struct sock *sk);
235
236 void *priv_ctx_tx;
237 void *priv_ctx_rx;
238
239 struct net_device *netdev;
240
241
242 struct cipher_context tx;
243 struct cipher_context rx;
244
245 struct scatterlist *partially_sent_record;
246 u16 partially_sent_offset;
247
248 bool in_tcp_sendpages;
249 bool pending_open_record_frags;
250
251 struct mutex tx_lock;
252
253
254 unsigned long flags;
255
256
257 struct proto *sk_proto;
258
259 void (*sk_destruct)(struct sock *sk);
260
261 union tls_crypto_context crypto_send;
262 union tls_crypto_context crypto_recv;
263
264 struct list_head list;
265 refcount_t refcount;
266 struct rcu_head rcu;
267};
268
269enum tls_offload_ctx_dir {
270 TLS_OFFLOAD_CTX_DIR_RX,
271 TLS_OFFLOAD_CTX_DIR_TX,
272};
273
274struct tlsdev_ops {
275 int (*tls_dev_add)(struct net_device *netdev, struct sock *sk,
276 enum tls_offload_ctx_dir direction,
277 struct tls_crypto_info *crypto_info,
278 u32 start_offload_tcp_sn);
279 void (*tls_dev_del)(struct net_device *netdev,
280 struct tls_context *ctx,
281 enum tls_offload_ctx_dir direction);
282 int (*tls_dev_resync)(struct net_device *netdev,
283 struct sock *sk, u32 seq, u8 *rcd_sn,
284 enum tls_offload_ctx_dir direction);
285};
286
287enum tls_offload_sync_type {
288 TLS_OFFLOAD_SYNC_TYPE_DRIVER_REQ = 0,
289 TLS_OFFLOAD_SYNC_TYPE_CORE_NEXT_HINT = 1,
290};
291
292#define TLS_DEVICE_RESYNC_NH_START_IVAL 2
293#define TLS_DEVICE_RESYNC_NH_MAX_IVAL 128
294
295struct tls_offload_context_rx {
296
297 struct tls_sw_context_rx sw;
298 enum tls_offload_sync_type resync_type;
299
300 u8 resync_nh_reset:1;
301
302 u8 resync_nh_do_now:1;
303 union {
304
305 struct {
306 atomic64_t resync_req;
307 };
308
309 struct {
310 u32 decrypted_failed;
311 u32 decrypted_tgt;
312 } resync_nh;
313 };
314 u8 driver_state[] __aligned(8);
315
316
317
318
319#define TLS_DRIVER_STATE_SIZE_RX 8
320};
321
322#define TLS_OFFLOAD_CONTEXT_SIZE_RX \
323 (sizeof(struct tls_offload_context_rx) + TLS_DRIVER_STATE_SIZE_RX)
324
325struct tls_context *tls_ctx_create(struct sock *sk);
326void tls_ctx_free(struct sock *sk, struct tls_context *ctx);
327void update_sk_prot(struct sock *sk, struct tls_context *ctx);
328
329int wait_on_pending_writer(struct sock *sk, long *timeo);
330int tls_sk_query(struct sock *sk, int optname, char __user *optval,
331 int __user *optlen);
332int tls_sk_attach(struct sock *sk, int optname, char __user *optval,
333 unsigned int optlen);
334
335int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx);
336void tls_sw_strparser_arm(struct sock *sk, struct tls_context *ctx);
337void tls_sw_strparser_done(struct tls_context *tls_ctx);
338int tls_sw_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
339int tls_sw_sendpage_locked(struct sock *sk, struct page *page,
340 int offset, size_t size, int flags);
341int tls_sw_sendpage(struct sock *sk, struct page *page,
342 int offset, size_t size, int flags);
343void tls_sw_cancel_work_tx(struct tls_context *tls_ctx);
344void tls_sw_release_resources_tx(struct sock *sk);
345void tls_sw_free_ctx_tx(struct tls_context *tls_ctx);
346void tls_sw_free_resources_rx(struct sock *sk);
347void tls_sw_release_resources_rx(struct sock *sk);
348void tls_sw_free_ctx_rx(struct tls_context *tls_ctx);
349int tls_sw_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
350 int nonblock, int flags, int *addr_len);
351bool tls_sw_stream_read(const struct sock *sk);
352ssize_t tls_sw_splice_read(struct socket *sock, loff_t *ppos,
353 struct pipe_inode_info *pipe,
354 size_t len, unsigned int flags);
355
356int tls_device_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
357int tls_device_sendpage(struct sock *sk, struct page *page,
358 int offset, size_t size, int flags);
359int tls_tx_records(struct sock *sk, int flags);
360
361struct tls_record_info *tls_get_record(struct tls_offload_context_tx *context,
362 u32 seq, u64 *p_record_sn);
363
364static inline bool tls_record_is_start_marker(struct tls_record_info *rec)
365{
366 return rec->len == 0;
367}
368
369static inline u32 tls_record_start_seq(struct tls_record_info *rec)
370{
371 return rec->end_seq - rec->len;
372}
373
374int tls_push_sg(struct sock *sk, struct tls_context *ctx,
375 struct scatterlist *sg, u16 first_offset,
376 int flags);
377int tls_push_partial_record(struct sock *sk, struct tls_context *ctx,
378 int flags);
379void tls_free_partial_record(struct sock *sk, struct tls_context *ctx);
380
381static inline struct tls_msg *tls_msg(struct sk_buff *skb)
382{
383 return (struct tls_msg *)strp_msg(skb);
384}
385
386static inline bool tls_is_partially_sent_record(struct tls_context *ctx)
387{
388 return !!ctx->partially_sent_record;
389}
390
391static inline bool tls_is_pending_open_record(struct tls_context *tls_ctx)
392{
393 return tls_ctx->pending_open_record_frags;
394}
395
396static inline bool is_tx_ready(struct tls_sw_context_tx *ctx)
397{
398 struct tls_rec *rec;
399
400 rec = list_first_entry(&ctx->tx_list, struct tls_rec, list);
401 if (!rec)
402 return false;
403
404 return READ_ONCE(rec->tx_ready);
405}
406
407static inline u16 tls_user_config(struct tls_context *ctx, bool tx)
408{
409 u16 config = tx ? ctx->tx_conf : ctx->rx_conf;
410
411 switch (config) {
412 case TLS_BASE:
413 return TLS_CONF_BASE;
414 case TLS_SW:
415 return TLS_CONF_SW;
416 case TLS_HW:
417 return TLS_CONF_HW;
418 case TLS_HW_RECORD:
419 return TLS_CONF_HW_RECORD;
420 }
421 return 0;
422}
423
424struct sk_buff *
425tls_validate_xmit_skb(struct sock *sk, struct net_device *dev,
426 struct sk_buff *skb);
427
428static inline bool tls_is_sk_tx_device_offloaded(struct sock *sk)
429{
430#ifdef CONFIG_SOCK_VALIDATE_XMIT
431 return sk_fullsock(sk) &&
432 (smp_load_acquire(&sk->sk_validate_xmit_skb) ==
433 &tls_validate_xmit_skb);
434#else
435 return false;
436#endif
437}
438
439static inline void tls_err_abort(struct sock *sk, int err)
440{
441 sk->sk_err = err;
442 sk->sk_error_report(sk);
443}
444
445static inline bool tls_bigint_increment(unsigned char *seq, int len)
446{
447 int i;
448
449 for (i = len - 1; i >= 0; i--) {
450 ++seq[i];
451 if (seq[i] != 0)
452 break;
453 }
454
455 return (i == -1);
456}
457
458static inline struct tls_context *tls_get_ctx(const struct sock *sk)
459{
460 struct inet_connection_sock *icsk = inet_csk(sk);
461
462
463
464
465 return (__force void *)icsk->icsk_ulp_data;
466}
467
468static inline void tls_advance_record_sn(struct sock *sk,
469 struct tls_prot_info *prot,
470 struct cipher_context *ctx)
471{
472 if (tls_bigint_increment(ctx->rec_seq, prot->rec_seq_size))
473 tls_err_abort(sk, EBADMSG);
474
475 if (prot->version != TLS_1_3_VERSION)
476 tls_bigint_increment(ctx->iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE,
477 prot->iv_size);
478}
479
480static inline void tls_fill_prepend(struct tls_context *ctx,
481 char *buf,
482 size_t plaintext_len,
483 unsigned char record_type,
484 int version)
485{
486 struct tls_prot_info *prot = &ctx->prot_info;
487 size_t pkt_len, iv_size = prot->iv_size;
488
489 pkt_len = plaintext_len + prot->tag_size;
490 if (version != TLS_1_3_VERSION) {
491 pkt_len += iv_size;
492
493 memcpy(buf + TLS_NONCE_OFFSET,
494 ctx->tx.iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE, iv_size);
495 }
496
497
498
499
500 buf[0] = version == TLS_1_3_VERSION ?
501 TLS_RECORD_TYPE_DATA : record_type;
502
503 buf[1] = TLS_1_2_VERSION_MINOR;
504 buf[2] = TLS_1_2_VERSION_MAJOR;
505
506 buf[3] = pkt_len >> 8;
507 buf[4] = pkt_len & 0xFF;
508}
509
510static inline void tls_make_aad(char *buf,
511 size_t size,
512 char *record_sequence,
513 int record_sequence_size,
514 unsigned char record_type,
515 int version)
516{
517 if (version != TLS_1_3_VERSION) {
518 memcpy(buf, record_sequence, record_sequence_size);
519 buf += 8;
520 } else {
521 size += TLS_CIPHER_AES_GCM_128_TAG_SIZE;
522 }
523
524 buf[0] = version == TLS_1_3_VERSION ?
525 TLS_RECORD_TYPE_DATA : record_type;
526 buf[1] = TLS_1_2_VERSION_MAJOR;
527 buf[2] = TLS_1_2_VERSION_MINOR;
528 buf[3] = size >> 8;
529 buf[4] = size & 0xFF;
530}
531
532static inline void xor_iv_with_seq(int version, char *iv, char *seq)
533{
534 int i;
535
536 if (version == TLS_1_3_VERSION) {
537 for (i = 0; i < 8; i++)
538 iv[i + 4] ^= seq[i];
539 }
540}
541
542
543static inline struct tls_sw_context_rx *tls_sw_ctx_rx(
544 const struct tls_context *tls_ctx)
545{
546 return (struct tls_sw_context_rx *)tls_ctx->priv_ctx_rx;
547}
548
549static inline struct tls_sw_context_tx *tls_sw_ctx_tx(
550 const struct tls_context *tls_ctx)
551{
552 return (struct tls_sw_context_tx *)tls_ctx->priv_ctx_tx;
553}
554
555static inline struct tls_offload_context_tx *
556tls_offload_ctx_tx(const struct tls_context *tls_ctx)
557{
558 return (struct tls_offload_context_tx *)tls_ctx->priv_ctx_tx;
559}
560
561static inline bool tls_sw_has_ctx_tx(const struct sock *sk)
562{
563 struct tls_context *ctx = tls_get_ctx(sk);
564
565 if (!ctx)
566 return false;
567 return !!tls_sw_ctx_tx(ctx);
568}
569
570void tls_sw_write_space(struct sock *sk, struct tls_context *ctx);
571void tls_device_write_space(struct sock *sk, struct tls_context *ctx);
572
573static inline struct tls_offload_context_rx *
574tls_offload_ctx_rx(const struct tls_context *tls_ctx)
575{
576 return (struct tls_offload_context_rx *)tls_ctx->priv_ctx_rx;
577}
578
579#if IS_ENABLED(CONFIG_TLS_DEVICE)
580static inline void *__tls_driver_ctx(struct tls_context *tls_ctx,
581 enum tls_offload_ctx_dir direction)
582{
583 if (direction == TLS_OFFLOAD_CTX_DIR_TX)
584 return tls_offload_ctx_tx(tls_ctx)->driver_state;
585 else
586 return tls_offload_ctx_rx(tls_ctx)->driver_state;
587}
588
589static inline void *
590tls_driver_ctx(const struct sock *sk, enum tls_offload_ctx_dir direction)
591{
592 return __tls_driver_ctx(tls_get_ctx(sk), direction);
593}
594#endif
595
596
597static inline void tls_offload_rx_resync_request(struct sock *sk, __be32 seq)
598{
599 struct tls_context *tls_ctx = tls_get_ctx(sk);
600 struct tls_offload_context_rx *rx_ctx = tls_offload_ctx_rx(tls_ctx);
601
602 atomic64_set(&rx_ctx->resync_req, ((u64)ntohl(seq) << 32) | 1);
603}
604
605static inline void
606tls_offload_rx_resync_set_type(struct sock *sk, enum tls_offload_sync_type type)
607{
608 struct tls_context *tls_ctx = tls_get_ctx(sk);
609
610 tls_offload_ctx_rx(tls_ctx)->resync_type = type;
611}
612
613
614static inline bool tls_offload_tx_resync_pending(struct sock *sk)
615{
616 struct tls_context *tls_ctx = tls_get_ctx(sk);
617 bool ret;
618
619 ret = test_bit(TLS_TX_SYNC_SCHED, &tls_ctx->flags);
620 smp_mb__after_atomic();
621 return ret;
622}
623
624int __net_init tls_proc_init(struct net *net);
625void __net_exit tls_proc_fini(struct net *net);
626
627int tls_proccess_cmsg(struct sock *sk, struct msghdr *msg,
628 unsigned char *record_type);
629int decrypt_skb(struct sock *sk, struct sk_buff *skb,
630 struct scatterlist *sgout);
631struct sk_buff *tls_encrypt_skb(struct sk_buff *skb);
632
633struct sk_buff *tls_validate_xmit_skb(struct sock *sk,
634 struct net_device *dev,
635 struct sk_buff *skb);
636
637int tls_sw_fallback_init(struct sock *sk,
638 struct tls_offload_context_tx *offload_ctx,
639 struct tls_crypto_info *crypto_info);
640
641#ifdef CONFIG_TLS_DEVICE
642void tls_device_init(void);
643void tls_device_cleanup(void);
644int tls_set_device_offload(struct sock *sk, struct tls_context *ctx);
645void tls_device_free_resources_tx(struct sock *sk);
646int tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx);
647void tls_device_offload_cleanup_rx(struct sock *sk);
648void tls_device_rx_resync_new_rec(struct sock *sk, u32 rcd_len, u32 seq);
649void tls_offload_tx_resync_request(struct sock *sk, u32 got_seq, u32 exp_seq);
650int tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx,
651 struct sk_buff *skb, struct strp_msg *rxm);
652#else
653static inline void tls_device_init(void) {}
654static inline void tls_device_cleanup(void) {}
655
656static inline int
657tls_set_device_offload(struct sock *sk, struct tls_context *ctx)
658{
659 return -EOPNOTSUPP;
660}
661
662static inline void tls_device_free_resources_tx(struct sock *sk) {}
663
664static inline int
665tls_set_device_offload_rx(struct sock *sk, struct tls_context *ctx)
666{
667 return -EOPNOTSUPP;
668}
669
670static inline void tls_device_offload_cleanup_rx(struct sock *sk) {}
671static inline void
672tls_device_rx_resync_new_rec(struct sock *sk, u32 rcd_len, u32 seq) {}
673
674static inline int
675tls_device_decrypted(struct sock *sk, struct tls_context *tls_ctx,
676 struct sk_buff *skb, struct strp_msg *rxm)
677{
678 return 0;
679}
680#endif
681#endif
682