LXR linux/drivers/crypto/chelsio/chtls/chtls.h

   1/*
   2 * Copyright (c) 2018 Chelsio Communications, Inc.
   3 *
   4 * This program is free software; you can redistribute it and/or modify
   5 * it under the terms of the GNU General Public License version 2 as
   6 * published by the Free Software Foundation.
   7 */
   8
   9#ifndef __CHTLS_H__
  10#define __CHTLS_H__
  11
  12#include <crypto/aes.h>
  13#include <crypto/algapi.h>
  14#include <crypto/hash.h>
  15#include <crypto/sha.h>
  16#include <crypto/authenc.h>
  17#include <crypto/ctr.h>
  18#include <crypto/gf128mul.h>
  19#include <crypto/internal/aead.h>
  20#include <crypto/null.h>
  21#include <crypto/internal/skcipher.h>
  22#include <crypto/aead.h>
  23#include <crypto/scatterwalk.h>
  24#include <crypto/internal/hash.h>
  25#include <linux/tls.h>
  26#include <net/tls.h>
  27
  28#include "t4fw_api.h"
  29#include "t4_msg.h"
  30#include "cxgb4.h"
  31#include "cxgb4_uld.h"
  32#include "l2t.h"
  33#include "chcr_algo.h"
  34#include "chcr_core.h"
  35#include "chcr_crypto.h"
  36
  37#define MAX_IVS_PAGE                    256
  38#define TLS_KEY_CONTEXT_SZ              64
  39#define CIPHER_BLOCK_SIZE               16
  40#define GCM_TAG_SIZE                    16
  41#define KEY_ON_MEM_SZ                   16
  42#define AEAD_EXPLICIT_DATA_SIZE         8
  43#define TLS_HEADER_LENGTH               5
  44#define SCMD_CIPH_MODE_AES_GCM          2
  45/* Any MFS size should work and come from openssl */
  46#define TLS_MFS                         16384
  47
  48#define RSS_HDR sizeof(struct rss_header)
  49#define TLS_WR_CPL_LEN \
  50        (sizeof(struct fw_tlstx_data_wr) + sizeof(struct cpl_tx_tls_sfo))
  51
  52enum {
  53        CHTLS_KEY_CONTEXT_DSGL,
  54        CHTLS_KEY_CONTEXT_IMM,
  55        CHTLS_KEY_CONTEXT_DDR,
  56};
  57
  58enum {
  59        CHTLS_LISTEN_START,
  60        CHTLS_LISTEN_STOP,
  61};
  62
  63/* Flags for return value of CPL message handlers */
  64enum {
  65        CPL_RET_BUF_DONE =    1,   /* buffer processing done */
  66        CPL_RET_BAD_MSG =     2,   /* bad CPL message */
  67        CPL_RET_UNKNOWN_TID = 4    /* unexpected unknown TID */
  68};
  69
  70#define LISTEN_INFO_HASH_SIZE 32
  71#define RSPQ_HASH_BITS 5
  72struct listen_info {
  73        struct listen_info *next;  /* Link to next entry */
  74        struct sock *sk;           /* The listening socket */
  75        unsigned int stid;         /* The server TID */
  76};
  77
  78enum {
  79        T4_LISTEN_START_PENDING,
  80        T4_LISTEN_STARTED
  81};
  82
  83enum csk_flags {
  84        CSK_CALLBACKS_CHKD,     /* socket callbacks have been sanitized */
  85        CSK_ABORT_REQ_RCVD,     /* received one ABORT_REQ_RSS message */
  86        CSK_TX_MORE_DATA,       /* sending ULP data; don't set SHOVE bit */
  87        CSK_TX_WAIT_IDLE,       /* suspend Tx until in-flight data is ACKed */
  88        CSK_ABORT_SHUTDOWN,     /* shouldn't send more abort requests */
  89        CSK_ABORT_RPL_PENDING,  /* expecting an abort reply */
  90        CSK_CLOSE_CON_REQUESTED,/* we've sent a close_conn_req */
  91        CSK_TX_DATA_SENT,       /* sent a TX_DATA WR on this connection */
  92        CSK_TX_FAILOVER,        /* Tx traffic failing over */
  93        CSK_UPDATE_RCV_WND,     /* Need to update rcv window */
  94        CSK_RST_ABORTED,        /* outgoing RST was aborted */
  95        CSK_TLS_HANDSHK,        /* TLS Handshake */
  96        CSK_CONN_INLINE,        /* Connection on HW */
  97};
  98
  99enum chtls_cdev_state {
 100        CHTLS_CDEV_STATE_UP = 1
 101};
 102
 103struct listen_ctx {
 104        struct sock *lsk;
 105        struct chtls_dev *cdev;
 106        struct sk_buff_head synq;
 107        u32 state;
 108};
 109
 110struct key_map {
 111        unsigned long *addr;
 112        unsigned int start;
 113        unsigned int available;
 114        unsigned int size;
 115        spinlock_t lock; /* lock for key id request from map */
 116} __packed;
 117
 118struct tls_scmd {
 119        u32 seqno_numivs;
 120        u32 ivgen_hdrlen;
 121};
 122
 123struct chtls_dev {
 124        struct tls_device tlsdev;
 125        struct list_head list;
 126        struct cxgb4_lld_info *lldi;
 127        struct pci_dev *pdev;
 128        struct listen_info *listen_hash_tab[LISTEN_INFO_HASH_SIZE];
 129        spinlock_t listen_lock; /* lock for listen list */
 130        struct net_device **ports;
 131        struct tid_info *tids;
 132        unsigned int pfvf;
 133        const unsigned short *mtus;
 134
 135        struct idr hwtid_idr;
 136        struct idr stid_idr;
 137
 138        spinlock_t idr_lock ____cacheline_aligned_in_smp;
 139
 140        struct net_device *egr_dev[NCHAN * 2];
 141        struct sk_buff *rspq_skb_cache[1 << RSPQ_HASH_BITS];
 142        struct sk_buff *askb;
 143
 144        struct sk_buff_head deferq;
 145        struct work_struct deferq_task;
 146
 147        struct list_head list_node;
 148        struct list_head rcu_node;
 149        struct list_head na_node;
 150        unsigned int send_page_order;
 151        int max_host_sndbuf;
 152        struct key_map kmap;
 153        unsigned int cdev_state;
 154};
 155
 156struct chtls_listen {
 157        struct chtls_dev *cdev;
 158        struct sock *sk;
 159};
 160
 161struct chtls_hws {
 162        struct sk_buff_head sk_recv_queue;
 163        u8 txqid;
 164        u8 ofld;
 165        u16 type;
 166        u16 rstate;
 167        u16 keyrpl;
 168        u16 pldlen;
 169        u16 rcvpld;
 170        u16 compute;
 171        u16 expansion;
 172        u16 keylen;
 173        u16 pdus;
 174        u16 adjustlen;
 175        u16 ivsize;
 176        u16 txleft;
 177        u32 mfs;
 178        s32 txkey;
 179        s32 rxkey;
 180        u32 fcplenmax;
 181        u32 copied_seq;
 182        u64 tx_seq_no;
 183        struct tls_scmd scmd;
 184        struct tls12_crypto_info_aes_gcm_128 crypto_info;
 185};
 186
 187struct chtls_sock {
 188        struct sock *sk;
 189        struct chtls_dev *cdev;
 190        struct l2t_entry *l2t_entry;    /* pointer to the L2T entry */
 191        struct net_device *egress_dev;  /* TX_CHAN for act open retry */
 192
 193        struct sk_buff_head txq;
 194        struct sk_buff *wr_skb_head;
 195        struct sk_buff *wr_skb_tail;
 196        struct sk_buff *ctrl_skb_cache;
 197        struct sk_buff *txdata_skb_cache; /* abort path messages */
 198        struct kref kref;
 199        unsigned long flags;
 200        u32 opt2;
 201        u32 wr_credits;
 202        u32 wr_unacked;
 203        u32 wr_max_credits;
 204        u32 wr_nondata;
 205        u32 hwtid;               /* TCP Control Block ID */
 206        u32 txq_idx;
 207        u32 rss_qid;
 208        u32 tid;
 209        u32 idr;
 210        u32 mss;
 211        u32 ulp_mode;
 212        u32 tx_chan;
 213        u32 rx_chan;
 214        u32 sndbuf;
 215        u32 txplen_max;
 216        u32 mtu_idx;           /* MTU table index */
 217        u32 smac_idx;
 218        u8 port_id;
 219        u8 tos;
 220        u16 resv2;
 221        u32 delack_mode;
 222        u32 delack_seq;
 223        u32 snd_win;
 224        u32 rcv_win;
 225
 226        void *passive_reap_next;        /* placeholder for passive */
 227        struct chtls_hws tlshws;
 228        struct synq {
 229                struct sk_buff *next;
 230                struct sk_buff *prev;
 231        } synq;
 232        struct listen_ctx *listen_ctx;
 233};
 234
 235struct tls_hdr {
 236        u8  type;
 237        u16 version;
 238        u16 length;
 239} __packed;
 240
 241struct tlsrx_cmp_hdr {
 242        u8  type;
 243        u16 version;
 244        u16 length;
 245
 246        u64 tls_seq;
 247        u16 reserved1;
 248        u8  res_to_mac_error;
 249} __packed;
 250
 251/* res_to_mac_error fields */
 252#define TLSRX_HDR_PKT_INT_ERROR_S   4
 253#define TLSRX_HDR_PKT_INT_ERROR_M   0x1
 254#define TLSRX_HDR_PKT_INT_ERROR_V(x) \
 255        ((x) << TLSRX_HDR_PKT_INT_ERROR_S)
 256#define TLSRX_HDR_PKT_INT_ERROR_G(x) \
 257        (((x) >> TLSRX_HDR_PKT_INT_ERROR_S) & TLSRX_HDR_PKT_INT_ERROR_M)
 258#define TLSRX_HDR_PKT_INT_ERROR_F   TLSRX_HDR_PKT_INT_ERROR_V(1U)
 259
 260#define TLSRX_HDR_PKT_SPP_ERROR_S        3
 261#define TLSRX_HDR_PKT_SPP_ERROR_M        0x1
 262#define TLSRX_HDR_PKT_SPP_ERROR_V(x)     ((x) << TLSRX_HDR_PKT_SPP_ERROR)
 263#define TLSRX_HDR_PKT_SPP_ERROR_G(x)     \
 264        (((x) >> TLSRX_HDR_PKT_SPP_ERROR_S) & TLSRX_HDR_PKT_SPP_ERROR_M)
 265#define TLSRX_HDR_PKT_SPP_ERROR_F        TLSRX_HDR_PKT_SPP_ERROR_V(1U)
 266
 267#define TLSRX_HDR_PKT_CCDX_ERROR_S       2
 268#define TLSRX_HDR_PKT_CCDX_ERROR_M       0x1
 269#define TLSRX_HDR_PKT_CCDX_ERROR_V(x)    ((x) << TLSRX_HDR_PKT_CCDX_ERROR_S)
 270#define TLSRX_HDR_PKT_CCDX_ERROR_G(x)    \
 271        (((x) >> TLSRX_HDR_PKT_CCDX_ERROR_S) & TLSRX_HDR_PKT_CCDX_ERROR_M)
 272#define TLSRX_HDR_PKT_CCDX_ERROR_F       TLSRX_HDR_PKT_CCDX_ERROR_V(1U)
 273
 274#define TLSRX_HDR_PKT_PAD_ERROR_S        1
 275#define TLSRX_HDR_PKT_PAD_ERROR_M        0x1
 276#define TLSRX_HDR_PKT_PAD_ERROR_V(x)     ((x) << TLSRX_HDR_PKT_PAD_ERROR_S)
 277#define TLSRX_HDR_PKT_PAD_ERROR_G(x)     \
 278        (((x) >> TLSRX_HDR_PKT_PAD_ERROR_S) & TLSRX_HDR_PKT_PAD_ERROR_M)
 279#define TLSRX_HDR_PKT_PAD_ERROR_F        TLSRX_HDR_PKT_PAD_ERROR_V(1U)
 280
 281#define TLSRX_HDR_PKT_MAC_ERROR_S        0
 282#define TLSRX_HDR_PKT_MAC_ERROR_M        0x1
 283#define TLSRX_HDR_PKT_MAC_ERROR_V(x)     ((x) << TLSRX_HDR_PKT_MAC_ERROR)
 284#define TLSRX_HDR_PKT_MAC_ERROR_G(x)     \
 285        (((x) >> S_TLSRX_HDR_PKT_MAC_ERROR_S) & TLSRX_HDR_PKT_MAC_ERROR_M)
 286#define TLSRX_HDR_PKT_MAC_ERROR_F        TLSRX_HDR_PKT_MAC_ERROR_V(1U)
 287
 288#define TLSRX_HDR_PKT_ERROR_M           0x1F
 289#define CONTENT_TYPE_ERROR              0x7F
 290
 291struct ulp_mem_rw {
 292        __be32 cmd;
 293        __be32 len16;             /* command length */
 294        __be32 dlen;              /* data length in 32-byte units */
 295        __be32 lock_addr;
 296};
 297
 298struct tls_key_wr {
 299        __be32 op_to_compl;
 300        __be32 flowid_len16;
 301        __be32 ftid;
 302        u8   reneg_to_write_rx;
 303        u8   protocol;
 304        __be16 mfs;
 305};
 306
 307struct tls_key_req {
 308        struct tls_key_wr wr;
 309        struct ulp_mem_rw req;
 310        struct ulptx_idata sc_imm;
 311};
 312
 313/*
 314 * This lives in skb->cb and is used to chain WRs in a linked list.
 315 */
 316struct wr_skb_cb {
 317        struct l2t_skb_cb l2t;          /* reserve space for l2t CB */
 318        struct sk_buff *next_wr;        /* next write request */
 319};
 320
 321/* Per-skb backlog handler.  Run when a socket's backlog is processed. */
 322struct blog_skb_cb {
 323        void (*backlog_rcv)(struct sock *sk, struct sk_buff *skb);
 324        struct chtls_dev *cdev;
 325};
 326
 327/*
 328 * Similar to tcp_skb_cb but with ULP elements added to support TLS,
 329 * etc.
 330 */
 331struct ulp_skb_cb {
 332        struct wr_skb_cb wr;            /* reserve space for write request */
 333        u16 flags;                      /* TCP-like flags */
 334        u8 psh;
 335        u8 ulp_mode;                    /* ULP mode/submode of sk_buff */
 336        u32 seq;                        /* TCP sequence number */
 337        union { /* ULP-specific fields */
 338                struct {
 339                        u8  type;
 340                        u8  ofld;
 341                        u8  iv;
 342                } tls;
 343        } ulp;
 344};
 345
 346#define ULP_SKB_CB(skb) ((struct ulp_skb_cb *)&((skb)->cb[0]))
 347#define BLOG_SKB_CB(skb) ((struct blog_skb_cb *)(skb)->cb)
 348
 349/*
 350 * Flags for ulp_skb_cb.flags.
 351 */
 352enum {
 353        ULPCB_FLAG_NEED_HDR  = 1 << 0,  /* packet needs a TX_DATA_WR header */
 354        ULPCB_FLAG_NO_APPEND = 1 << 1,  /* don't grow this skb */
 355        ULPCB_FLAG_BARRIER   = 1 << 2,  /* set TX_WAIT_IDLE after sending */
 356        ULPCB_FLAG_HOLD      = 1 << 3,  /* skb not ready for Tx yet */
 357        ULPCB_FLAG_COMPL     = 1 << 4,  /* request WR completion */
 358        ULPCB_FLAG_URG       = 1 << 5,  /* urgent data */
 359        ULPCB_FLAG_TLS_HDR   = 1 << 6,  /* payload with tls hdr */
 360        ULPCB_FLAG_NO_HDR    = 1 << 7,  /* not a ofld wr */
 361};
 362
 363/* The ULP mode/submode of an skbuff */
 364#define skb_ulp_mode(skb)  (ULP_SKB_CB(skb)->ulp_mode)
 365#define TCP_PAGE(sk)   (sk->sk_frag.page)
 366#define TCP_OFF(sk)    (sk->sk_frag.offset)
 367
 368static inline struct chtls_dev *to_chtls_dev(struct tls_device *tlsdev)
 369{
 370        return container_of(tlsdev, struct chtls_dev, tlsdev);
 371}
 372
 373static inline void csk_set_flag(struct chtls_sock *csk,
 374                                enum csk_flags flag)
 375{
 376        __set_bit(flag, &csk->flags);
 377}
 378
 379static inline void csk_reset_flag(struct chtls_sock *csk,
 380                                  enum csk_flags flag)
 381{
 382        __clear_bit(flag, &csk->flags);
 383}
 384
 385static inline bool csk_conn_inline(const struct chtls_sock *csk)
 386{
 387        return test_bit(CSK_CONN_INLINE, &csk->flags);
 388}
 389
 390static inline int csk_flag(const struct sock *sk, enum csk_flags flag)
 391{
 392        struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
 393
 394        if (!csk_conn_inline(csk))
 395                return 0;
 396        return test_bit(flag, &csk->flags);
 397}
 398
 399static inline int csk_flag_nochk(const struct chtls_sock *csk,
 400                                 enum csk_flags flag)
 401{
 402        return test_bit(flag, &csk->flags);
 403}
 404
 405static inline void *cplhdr(struct sk_buff *skb)
 406{
 407        return skb->data;
 408}
 409
 410static inline int is_neg_adv(unsigned int status)
 411{
 412        return status == CPL_ERR_RTX_NEG_ADVICE ||
 413               status == CPL_ERR_KEEPALV_NEG_ADVICE ||
 414               status == CPL_ERR_PERSIST_NEG_ADVICE;
 415}
 416
 417static inline void process_cpl_msg(void (*fn)(struct sock *, struct sk_buff *),
 418                                   struct sock *sk,
 419                                   struct sk_buff *skb)
 420{
 421        skb_reset_mac_header(skb);
 422        skb_reset_network_header(skb);
 423        skb_reset_transport_header(skb);
 424
 425        bh_lock_sock(sk);
 426        if (unlikely(sock_owned_by_user(sk))) {
 427                BLOG_SKB_CB(skb)->backlog_rcv = fn;
 428                __sk_add_backlog(sk, skb);
 429        } else {
 430                fn(sk, skb);
 431        }
 432        bh_unlock_sock(sk);
 433}
 434
 435static inline void chtls_sock_free(struct kref *ref)
 436{
 437        struct chtls_sock *csk = container_of(ref, struct chtls_sock,
 438                                              kref);
 439        kfree(csk);
 440}
 441
 442static inline void __chtls_sock_put(const char *fn, struct chtls_sock *csk)
 443{
 444        kref_put(&csk->kref, chtls_sock_free);
 445}
 446
 447static inline void __chtls_sock_get(const char *fn,
 448                                    struct chtls_sock *csk)
 449{
 450        kref_get(&csk->kref);
 451}
 452
 453static inline void send_or_defer(struct sock *sk, struct tcp_sock *tp,
 454                                 struct sk_buff *skb, int through_l2t)
 455{
 456        struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
 457
 458        if (through_l2t) {
 459                /* send through L2T */
 460                cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry);
 461        } else {
 462                /* send directly */
 463                cxgb4_ofld_send(csk->egress_dev, skb);
 464        }
 465}
 466
 467typedef int (*chtls_handler_func)(struct chtls_dev *, struct sk_buff *);
 468extern chtls_handler_func chtls_handlers[NUM_CPL_CMDS];
 469void chtls_install_cpl_ops(struct sock *sk);
 470int chtls_init_kmap(struct chtls_dev *cdev, struct cxgb4_lld_info *lldi);
 471void chtls_listen_stop(struct chtls_dev *cdev, struct sock *sk);
 472int chtls_listen_start(struct chtls_dev *cdev, struct sock *sk);
 473void chtls_close(struct sock *sk, long timeout);
 474int chtls_disconnect(struct sock *sk, int flags);
 475void chtls_shutdown(struct sock *sk, int how);
 476void chtls_destroy_sock(struct sock *sk);
 477int chtls_sendmsg(struct sock *sk, struct msghdr *msg, size_t size);
 478int chtls_recvmsg(struct sock *sk, struct msghdr *msg,
 479                  size_t len, int nonblock, int flags, int *addr_len);
 480int chtls_sendpage(struct sock *sk, struct page *page,
 481                   int offset, size_t size, int flags);
 482int send_tx_flowc_wr(struct sock *sk, int compl,
 483                     u32 snd_nxt, u32 rcv_nxt);
 484void chtls_tcp_push(struct sock *sk, int flags);
 485int chtls_push_frames(struct chtls_sock *csk, int comp);
 486int chtls_set_tcb_tflag(struct sock *sk, unsigned int bit_pos, int val);
 487int chtls_setkey(struct chtls_sock *csk, u32 keylen, u32 mode);
 488void skb_entail(struct sock *sk, struct sk_buff *skb, int flags);
 489unsigned int keyid_to_addr(int start_addr, int keyid);
 490void free_tls_keyid(struct sock *sk);
 491#endif
 492