1
2
3
4
5
6#ifndef __CHTLS_H__
7#define __CHTLS_H__
8
9#include <crypto/aes.h>
10#include <crypto/algapi.h>
11#include <crypto/hash.h>
12#include <crypto/sha1.h>
13#include <crypto/sha2.h>
14#include <crypto/authenc.h>
15#include <crypto/ctr.h>
16#include <crypto/gf128mul.h>
17#include <crypto/internal/aead.h>
18#include <crypto/null.h>
19#include <crypto/internal/skcipher.h>
20#include <crypto/aead.h>
21#include <crypto/scatterwalk.h>
22#include <crypto/internal/hash.h>
23#include <linux/tls.h>
24#include <net/tls.h>
25#include <net/tls_toe.h>
26
27#include "t4fw_api.h"
28#include "t4_msg.h"
29#include "cxgb4.h"
30#include "cxgb4_uld.h"
31#include "l2t.h"
32#include "chcr_algo.h"
33#include "chcr_core.h"
34#include "chcr_crypto.h"
35
36#define CHTLS_DRV_VERSION "1.0.0.0-ko"
37
38#define TLS_KEYCTX_RXFLIT_CNT_S 24
39#define TLS_KEYCTX_RXFLIT_CNT_V(x) ((x) << TLS_KEYCTX_RXFLIT_CNT_S)
40
41#define TLS_KEYCTX_RXPROT_VER_S 20
42#define TLS_KEYCTX_RXPROT_VER_M 0xf
43#define TLS_KEYCTX_RXPROT_VER_V(x) ((x) << TLS_KEYCTX_RXPROT_VER_S)
44
45#define TLS_KEYCTX_RXCIPH_MODE_S 16
46#define TLS_KEYCTX_RXCIPH_MODE_M 0xf
47#define TLS_KEYCTX_RXCIPH_MODE_V(x) ((x) << TLS_KEYCTX_RXCIPH_MODE_S)
48
49#define TLS_KEYCTX_RXAUTH_MODE_S 12
50#define TLS_KEYCTX_RXAUTH_MODE_M 0xf
51#define TLS_KEYCTX_RXAUTH_MODE_V(x) ((x) << TLS_KEYCTX_RXAUTH_MODE_S)
52
53#define TLS_KEYCTX_RXCIAU_CTRL_S 11
54#define TLS_KEYCTX_RXCIAU_CTRL_V(x) ((x) << TLS_KEYCTX_RXCIAU_CTRL_S)
55
56#define TLS_KEYCTX_RX_SEQCTR_S 9
57#define TLS_KEYCTX_RX_SEQCTR_M 0x3
58#define TLS_KEYCTX_RX_SEQCTR_V(x) ((x) << TLS_KEYCTX_RX_SEQCTR_S)
59
60#define TLS_KEYCTX_RX_VALID_S 8
61#define TLS_KEYCTX_RX_VALID_V(x) ((x) << TLS_KEYCTX_RX_VALID_S)
62
63#define TLS_KEYCTX_RXCK_SIZE_S 3
64#define TLS_KEYCTX_RXCK_SIZE_M 0x7
65#define TLS_KEYCTX_RXCK_SIZE_V(x) ((x) << TLS_KEYCTX_RXCK_SIZE_S)
66
67#define TLS_KEYCTX_RXMK_SIZE_S 0
68#define TLS_KEYCTX_RXMK_SIZE_M 0x7
69#define TLS_KEYCTX_RXMK_SIZE_V(x) ((x) << TLS_KEYCTX_RXMK_SIZE_S)
70
71#define KEYCTX_TX_WR_IV_S 55
72#define KEYCTX_TX_WR_IV_M 0x1ffULL
73#define KEYCTX_TX_WR_IV_V(x) ((x) << KEYCTX_TX_WR_IV_S)
74#define KEYCTX_TX_WR_IV_G(x) \
75 (((x) >> KEYCTX_TX_WR_IV_S) & KEYCTX_TX_WR_IV_M)
76
77#define KEYCTX_TX_WR_AAD_S 47
78#define KEYCTX_TX_WR_AAD_M 0xffULL
79#define KEYCTX_TX_WR_AAD_V(x) ((x) << KEYCTX_TX_WR_AAD_S)
80#define KEYCTX_TX_WR_AAD_G(x) (((x) >> KEYCTX_TX_WR_AAD_S) & \
81 KEYCTX_TX_WR_AAD_M)
82
83#define KEYCTX_TX_WR_AADST_S 39
84#define KEYCTX_TX_WR_AADST_M 0xffULL
85#define KEYCTX_TX_WR_AADST_V(x) ((x) << KEYCTX_TX_WR_AADST_S)
86#define KEYCTX_TX_WR_AADST_G(x) \
87 (((x) >> KEYCTX_TX_WR_AADST_S) & KEYCTX_TX_WR_AADST_M)
88
89#define KEYCTX_TX_WR_CIPHER_S 30
90#define KEYCTX_TX_WR_CIPHER_M 0x1ffULL
91#define KEYCTX_TX_WR_CIPHER_V(x) ((x) << KEYCTX_TX_WR_CIPHER_S)
92#define KEYCTX_TX_WR_CIPHER_G(x) \
93 (((x) >> KEYCTX_TX_WR_CIPHER_S) & KEYCTX_TX_WR_CIPHER_M)
94
95#define KEYCTX_TX_WR_CIPHERST_S 23
96#define KEYCTX_TX_WR_CIPHERST_M 0x7f
97#define KEYCTX_TX_WR_CIPHERST_V(x) ((x) << KEYCTX_TX_WR_CIPHERST_S)
98#define KEYCTX_TX_WR_CIPHERST_G(x) \
99 (((x) >> KEYCTX_TX_WR_CIPHERST_S) & KEYCTX_TX_WR_CIPHERST_M)
100
101#define KEYCTX_TX_WR_AUTH_S 14
102#define KEYCTX_TX_WR_AUTH_M 0x1ff
103#define KEYCTX_TX_WR_AUTH_V(x) ((x) << KEYCTX_TX_WR_AUTH_S)
104#define KEYCTX_TX_WR_AUTH_G(x) \
105 (((x) >> KEYCTX_TX_WR_AUTH_S) & KEYCTX_TX_WR_AUTH_M)
106
107#define KEYCTX_TX_WR_AUTHST_S 7
108#define KEYCTX_TX_WR_AUTHST_M 0x7f
109#define KEYCTX_TX_WR_AUTHST_V(x) ((x) << KEYCTX_TX_WR_AUTHST_S)
110#define KEYCTX_TX_WR_AUTHST_G(x) \
111 (((x) >> KEYCTX_TX_WR_AUTHST_S) & KEYCTX_TX_WR_AUTHST_M)
112
113#define KEYCTX_TX_WR_AUTHIN_S 0
114#define KEYCTX_TX_WR_AUTHIN_M 0x7f
115#define KEYCTX_TX_WR_AUTHIN_V(x) ((x) << KEYCTX_TX_WR_AUTHIN_S)
116#define KEYCTX_TX_WR_AUTHIN_G(x) \
117 (((x) >> KEYCTX_TX_WR_AUTHIN_S) & KEYCTX_TX_WR_AUTHIN_M)
118
119struct sge_opaque_hdr {
120 void *dev;
121 dma_addr_t addr[MAX_SKB_FRAGS + 1];
122};
123
124#define MAX_IVS_PAGE 256
125#define TLS_KEY_CONTEXT_SZ 64
126#define CIPHER_BLOCK_SIZE 16
127#define GCM_TAG_SIZE 16
128#define KEY_ON_MEM_SZ 16
129#define AEAD_EXPLICIT_DATA_SIZE 8
130#define TLS_HEADER_LENGTH 5
131#define SCMD_CIPH_MODE_AES_GCM 2
132
133#define TLS_MFS 16384
134
135#define RSS_HDR sizeof(struct rss_header)
136#define TLS_WR_CPL_LEN \
137 (sizeof(struct fw_tlstx_data_wr) + sizeof(struct cpl_tx_tls_sfo))
138
139enum {
140 CHTLS_KEY_CONTEXT_DSGL,
141 CHTLS_KEY_CONTEXT_IMM,
142 CHTLS_KEY_CONTEXT_DDR,
143};
144
145enum {
146 CHTLS_LISTEN_START,
147 CHTLS_LISTEN_STOP,
148};
149
150
151enum {
152 CPL_RET_BUF_DONE = 1,
153 CPL_RET_BAD_MSG = 2,
154 CPL_RET_UNKNOWN_TID = 4
155};
156
157#define LISTEN_INFO_HASH_SIZE 32
158#define RSPQ_HASH_BITS 5
159struct listen_info {
160 struct listen_info *next;
161 struct sock *sk;
162 unsigned int stid;
163};
164
165enum {
166 T4_LISTEN_START_PENDING,
167 T4_LISTEN_STARTED
168};
169
170enum csk_flags {
171 CSK_CALLBACKS_CHKD,
172 CSK_ABORT_REQ_RCVD,
173 CSK_TX_MORE_DATA,
174 CSK_TX_WAIT_IDLE,
175 CSK_ABORT_SHUTDOWN,
176 CSK_ABORT_RPL_PENDING,
177 CSK_CLOSE_CON_REQUESTED,
178 CSK_TX_DATA_SENT,
179 CSK_TX_FAILOVER,
180 CSK_UPDATE_RCV_WND,
181 CSK_RST_ABORTED,
182 CSK_TLS_HANDSHK,
183 CSK_CONN_INLINE,
184};
185
186enum chtls_cdev_state {
187 CHTLS_CDEV_STATE_UP = 1
188};
189
190struct listen_ctx {
191 struct sock *lsk;
192 struct chtls_dev *cdev;
193 struct sk_buff_head synq;
194 u32 state;
195};
196
197struct key_map {
198 unsigned long *addr;
199 unsigned int start;
200 unsigned int available;
201 unsigned int size;
202 spinlock_t lock;
203} __packed;
204
205struct tls_scmd {
206 u32 seqno_numivs;
207 u32 ivgen_hdrlen;
208};
209
210struct chtls_dev {
211 struct tls_toe_device tlsdev;
212 struct list_head list;
213 struct cxgb4_lld_info *lldi;
214 struct pci_dev *pdev;
215 struct listen_info *listen_hash_tab[LISTEN_INFO_HASH_SIZE];
216 spinlock_t listen_lock;
217 struct net_device **ports;
218 struct tid_info *tids;
219 unsigned int pfvf;
220 const unsigned short *mtus;
221
222 struct idr hwtid_idr;
223 struct idr stid_idr;
224
225 spinlock_t idr_lock ____cacheline_aligned_in_smp;
226
227 struct net_device *egr_dev[NCHAN * 2];
228 struct sk_buff *rspq_skb_cache[1 << RSPQ_HASH_BITS];
229 struct sk_buff *askb;
230
231 struct sk_buff_head deferq;
232 struct work_struct deferq_task;
233
234 struct list_head list_node;
235 struct list_head rcu_node;
236 struct list_head na_node;
237 unsigned int send_page_order;
238 int max_host_sndbuf;
239 u32 round_robin_cnt;
240 struct key_map kmap;
241 unsigned int cdev_state;
242};
243
244struct chtls_listen {
245 struct chtls_dev *cdev;
246 struct sock *sk;
247};
248
249struct chtls_hws {
250 struct sk_buff_head sk_recv_queue;
251 u8 txqid;
252 u8 ofld;
253 u16 type;
254 u16 rstate;
255 u16 keyrpl;
256 u16 pldlen;
257 u16 rcvpld;
258 u16 compute;
259 u16 expansion;
260 u16 keylen;
261 u16 pdus;
262 u16 adjustlen;
263 u16 ivsize;
264 u16 txleft;
265 u32 mfs;
266 s32 txkey;
267 s32 rxkey;
268 u32 fcplenmax;
269 u32 copied_seq;
270 u64 tx_seq_no;
271 struct tls_scmd scmd;
272 union {
273 struct tls12_crypto_info_aes_gcm_128 aes_gcm_128;
274 struct tls12_crypto_info_aes_gcm_256 aes_gcm_256;
275 } crypto_info;
276};
277
278struct chtls_sock {
279 struct sock *sk;
280 struct chtls_dev *cdev;
281 struct l2t_entry *l2t_entry;
282 struct net_device *egress_dev;
283
284 struct sk_buff_head txq;
285 struct sk_buff *wr_skb_head;
286 struct sk_buff *wr_skb_tail;
287 struct sk_buff *ctrl_skb_cache;
288 struct sk_buff *txdata_skb_cache;
289 struct kref kref;
290 unsigned long flags;
291 u32 opt2;
292 u32 wr_credits;
293 u32 wr_unacked;
294 u32 wr_max_credits;
295 u32 wr_nondata;
296 u32 hwtid;
297 u32 txq_idx;
298 u32 rss_qid;
299 u32 tid;
300 u32 idr;
301 u32 mss;
302 u32 ulp_mode;
303 u32 tx_chan;
304 u32 rx_chan;
305 u32 sndbuf;
306 u32 txplen_max;
307 u32 mtu_idx;
308 u32 smac_idx;
309 u8 port_id;
310 u8 tos;
311 u16 resv2;
312 u32 delack_mode;
313 u32 delack_seq;
314 u32 snd_win;
315 u32 rcv_win;
316
317 void *passive_reap_next;
318 struct chtls_hws tlshws;
319 struct synq {
320 struct sk_buff *next;
321 struct sk_buff *prev;
322 } synq;
323 struct listen_ctx *listen_ctx;
324};
325
326struct tls_hdr {
327 u8 type;
328 u16 version;
329 u16 length;
330} __packed;
331
332struct tlsrx_cmp_hdr {
333 u8 type;
334 u16 version;
335 u16 length;
336
337 u64 tls_seq;
338 u16 reserved1;
339 u8 res_to_mac_error;
340} __packed;
341
342
343#define TLSRX_HDR_PKT_INT_ERROR_S 4
344#define TLSRX_HDR_PKT_INT_ERROR_M 0x1
345#define TLSRX_HDR_PKT_INT_ERROR_V(x) \
346 ((x) << TLSRX_HDR_PKT_INT_ERROR_S)
347#define TLSRX_HDR_PKT_INT_ERROR_G(x) \
348 (((x) >> TLSRX_HDR_PKT_INT_ERROR_S) & TLSRX_HDR_PKT_INT_ERROR_M)
349#define TLSRX_HDR_PKT_INT_ERROR_F TLSRX_HDR_PKT_INT_ERROR_V(1U)
350
351#define TLSRX_HDR_PKT_SPP_ERROR_S 3
352#define TLSRX_HDR_PKT_SPP_ERROR_M 0x1
353#define TLSRX_HDR_PKT_SPP_ERROR_V(x) ((x) << TLSRX_HDR_PKT_SPP_ERROR)
354#define TLSRX_HDR_PKT_SPP_ERROR_G(x) \
355 (((x) >> TLSRX_HDR_PKT_SPP_ERROR_S) & TLSRX_HDR_PKT_SPP_ERROR_M)
356#define TLSRX_HDR_PKT_SPP_ERROR_F TLSRX_HDR_PKT_SPP_ERROR_V(1U)
357
358#define TLSRX_HDR_PKT_CCDX_ERROR_S 2
359#define TLSRX_HDR_PKT_CCDX_ERROR_M 0x1
360#define TLSRX_HDR_PKT_CCDX_ERROR_V(x) ((x) << TLSRX_HDR_PKT_CCDX_ERROR_S)
361#define TLSRX_HDR_PKT_CCDX_ERROR_G(x) \
362 (((x) >> TLSRX_HDR_PKT_CCDX_ERROR_S) & TLSRX_HDR_PKT_CCDX_ERROR_M)
363#define TLSRX_HDR_PKT_CCDX_ERROR_F TLSRX_HDR_PKT_CCDX_ERROR_V(1U)
364
365#define TLSRX_HDR_PKT_PAD_ERROR_S 1
366#define TLSRX_HDR_PKT_PAD_ERROR_M 0x1
367#define TLSRX_HDR_PKT_PAD_ERROR_V(x) ((x) << TLSRX_HDR_PKT_PAD_ERROR_S)
368#define TLSRX_HDR_PKT_PAD_ERROR_G(x) \
369 (((x) >> TLSRX_HDR_PKT_PAD_ERROR_S) & TLSRX_HDR_PKT_PAD_ERROR_M)
370#define TLSRX_HDR_PKT_PAD_ERROR_F TLSRX_HDR_PKT_PAD_ERROR_V(1U)
371
372#define TLSRX_HDR_PKT_MAC_ERROR_S 0
373#define TLSRX_HDR_PKT_MAC_ERROR_M 0x1
374#define TLSRX_HDR_PKT_MAC_ERROR_V(x) ((x) << TLSRX_HDR_PKT_MAC_ERROR)
375#define TLSRX_HDR_PKT_MAC_ERROR_G(x) \
376 (((x) >> S_TLSRX_HDR_PKT_MAC_ERROR_S) & TLSRX_HDR_PKT_MAC_ERROR_M)
377#define TLSRX_HDR_PKT_MAC_ERROR_F TLSRX_HDR_PKT_MAC_ERROR_V(1U)
378
379#define TLSRX_HDR_PKT_ERROR_M 0x1F
380#define CONTENT_TYPE_ERROR 0x7F
381
382struct ulp_mem_rw {
383 __be32 cmd;
384 __be32 len16;
385 __be32 dlen;
386 __be32 lock_addr;
387};
388
389struct tls_key_wr {
390 __be32 op_to_compl;
391 __be32 flowid_len16;
392 __be32 ftid;
393 u8 reneg_to_write_rx;
394 u8 protocol;
395 __be16 mfs;
396};
397
398struct tls_key_req {
399 struct tls_key_wr wr;
400 struct ulp_mem_rw req;
401 struct ulptx_idata sc_imm;
402};
403
404
405
406
407struct wr_skb_cb {
408 struct l2t_skb_cb l2t;
409 struct sk_buff *next_wr;
410};
411
412
413struct blog_skb_cb {
414 void (*backlog_rcv)(struct sock *sk, struct sk_buff *skb);
415 struct chtls_dev *cdev;
416};
417
418
419
420
421
422struct ulp_skb_cb {
423 struct wr_skb_cb wr;
424 u16 flags;
425 u8 psh;
426 u8 ulp_mode;
427 u32 seq;
428 union {
429 struct {
430 u8 type;
431 u8 ofld;
432 u8 iv;
433 } tls;
434 } ulp;
435};
436
437#define ULP_SKB_CB(skb) ((struct ulp_skb_cb *)&((skb)->cb[0]))
438#define BLOG_SKB_CB(skb) ((struct blog_skb_cb *)(skb)->cb)
439
440
441
442
443enum {
444 ULPCB_FLAG_NEED_HDR = 1 << 0,
445 ULPCB_FLAG_NO_APPEND = 1 << 1,
446 ULPCB_FLAG_BARRIER = 1 << 2,
447 ULPCB_FLAG_HOLD = 1 << 3,
448 ULPCB_FLAG_COMPL = 1 << 4,
449 ULPCB_FLAG_URG = 1 << 5,
450 ULPCB_FLAG_TLS_HDR = 1 << 6,
451 ULPCB_FLAG_NO_HDR = 1 << 7,
452};
453
454
455#define skb_ulp_mode(skb) (ULP_SKB_CB(skb)->ulp_mode)
456#define TCP_PAGE(sk) (sk->sk_frag.page)
457#define TCP_OFF(sk) (sk->sk_frag.offset)
458
459static inline struct chtls_dev *to_chtls_dev(struct tls_toe_device *tlsdev)
460{
461 return container_of(tlsdev, struct chtls_dev, tlsdev);
462}
463
464static inline void csk_set_flag(struct chtls_sock *csk,
465 enum csk_flags flag)
466{
467 __set_bit(flag, &csk->flags);
468}
469
470static inline void csk_reset_flag(struct chtls_sock *csk,
471 enum csk_flags flag)
472{
473 __clear_bit(flag, &csk->flags);
474}
475
476static inline bool csk_conn_inline(const struct chtls_sock *csk)
477{
478 return test_bit(CSK_CONN_INLINE, &csk->flags);
479}
480
481static inline int csk_flag(const struct sock *sk, enum csk_flags flag)
482{
483 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
484
485 if (!csk_conn_inline(csk))
486 return 0;
487 return test_bit(flag, &csk->flags);
488}
489
490static inline int csk_flag_nochk(const struct chtls_sock *csk,
491 enum csk_flags flag)
492{
493 return test_bit(flag, &csk->flags);
494}
495
496static inline void *cplhdr(struct sk_buff *skb)
497{
498 return skb->data;
499}
500
501static inline int is_neg_adv(unsigned int status)
502{
503 return status == CPL_ERR_RTX_NEG_ADVICE ||
504 status == CPL_ERR_KEEPALV_NEG_ADVICE ||
505 status == CPL_ERR_PERSIST_NEG_ADVICE;
506}
507
508static inline void process_cpl_msg(void (*fn)(struct sock *, struct sk_buff *),
509 struct sock *sk,
510 struct sk_buff *skb)
511{
512 skb_reset_mac_header(skb);
513 skb_reset_network_header(skb);
514 skb_reset_transport_header(skb);
515
516 bh_lock_sock(sk);
517 if (unlikely(sock_owned_by_user(sk))) {
518 BLOG_SKB_CB(skb)->backlog_rcv = fn;
519 __sk_add_backlog(sk, skb);
520 } else {
521 fn(sk, skb);
522 }
523 bh_unlock_sock(sk);
524}
525
526static inline void chtls_sock_free(struct kref *ref)
527{
528 struct chtls_sock *csk = container_of(ref, struct chtls_sock,
529 kref);
530 kfree(csk);
531}
532
533static inline void __chtls_sock_put(const char *fn, struct chtls_sock *csk)
534{
535 kref_put(&csk->kref, chtls_sock_free);
536}
537
538static inline void __chtls_sock_get(const char *fn,
539 struct chtls_sock *csk)
540{
541 kref_get(&csk->kref);
542}
543
544static inline void send_or_defer(struct sock *sk, struct tcp_sock *tp,
545 struct sk_buff *skb, int through_l2t)
546{
547 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
548
549 if (through_l2t) {
550
551 cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry);
552 } else {
553
554 cxgb4_ofld_send(csk->egress_dev, skb);
555 }
556}
557
558typedef int (*chtls_handler_func)(struct chtls_dev *, struct sk_buff *);
559extern chtls_handler_func chtls_handlers[NUM_CPL_CMDS];
560void chtls_install_cpl_ops(struct sock *sk);
561int chtls_init_kmap(struct chtls_dev *cdev, struct cxgb4_lld_info *lldi);
562void chtls_listen_stop(struct chtls_dev *cdev, struct sock *sk);
563int chtls_listen_start(struct chtls_dev *cdev, struct sock *sk);
564void chtls_close(struct sock *sk, long timeout);
565int chtls_disconnect(struct sock *sk, int flags);
566void chtls_shutdown(struct sock *sk, int how);
567void chtls_destroy_sock(struct sock *sk);
568int chtls_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
569int chtls_recvmsg(struct sock *sk, struct msghdr *msg,
570 size_t len, int nonblock, int flags, int *addr_len);
571int chtls_sendpage(struct sock *sk, struct page *page,
572 int offset, size_t size, int flags);
573int send_tx_flowc_wr(struct sock *sk, int compl,
574 u32 snd_nxt, u32 rcv_nxt);
575void chtls_tcp_push(struct sock *sk, int flags);
576int chtls_push_frames(struct chtls_sock *csk, int comp);
577int chtls_set_tcb_tflag(struct sock *sk, unsigned int bit_pos, int val);
578void chtls_set_tcb_field_rpl_skb(struct sock *sk, u16 word,
579 u64 mask, u64 val, u8 cookie,
580 int through_l2t);
581int chtls_setkey(struct chtls_sock *csk, u32 keylen, u32 mode, int cipher_type);
582void chtls_set_quiesce_ctrl(struct sock *sk, int val);
583void skb_entail(struct sock *sk, struct sk_buff *skb, int flags);
584unsigned int keyid_to_addr(int start_addr, int keyid);
585void free_tls_keyid(struct sock *sk);
586#endif
587
