LXR linux/drivers/staging/lustre/lnet/klnds/socklnd/socklnd.h

   1/*
   2 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
   3 *
   4 * Copyright (c) 2011, 2012, Intel Corporation.
   5 *
   6 *   Author: Zach Brown <zab@zabbo.net>
   7 *   Author: Peter J. Braam <braam@clusterfs.com>
   8 *   Author: Phil Schwan <phil@clusterfs.com>
   9 *   Author: Eric Barton <eric@bartonsoftware.com>
  10 *
  11 *   This file is part of Lustre, http://www.lustre.org
  12 *
  13 *   Portals is free software; you can redistribute it and/or
  14 *   modify it under the terms of version 2 of the GNU General Public
  15 *   License as published by the Free Software Foundation.
  16 *
  17 *   Portals is distributed in the hope that it will be useful,
  18 *   but WITHOUT ANY WARRANTY; without even the implied warranty of
  19 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  20 *   GNU General Public License for more details.
  21 *
  22 */
  23
  24#ifndef _SOCKLND_SOCKLND_H_
  25#define _SOCKLND_SOCKLND_H_
  26
  27#define DEBUG_PORTAL_ALLOC
  28#define DEBUG_SUBSYSTEM S_LND
  29
  30#include <linux/crc32.h>
  31#include <linux/errno.h>
  32#include <linux/if.h>
  33#include <linux/init.h>
  34#include <linux/kernel.h>
  35#include <linux/kmod.h>
  36#include <linux/list.h>
  37#include <linux/mm.h>
  38#include <linux/module.h>
  39#include <linux/stat.h>
  40#include <linux/string.h>
  41#include <linux/syscalls.h>
  42#include <linux/sysctl.h>
  43#include <linux/uio.h>
  44#include <linux/unistd.h>
  45#include <asm/irq.h>
  46#include <net/sock.h>
  47#include <net/tcp.h>
  48
  49#include "../../../include/linux/libcfs/libcfs.h"
  50#include "../../../include/linux/lnet/lnet.h"
  51#include "../../../include/linux/lnet/lib-lnet.h"
  52#include "../../../include/linux/lnet/socklnd.h"
  53
  54/* assume one thread for each connection type */
  55#define SOCKNAL_NSCHEDS         3
  56#define SOCKNAL_NSCHEDS_HIGH    (SOCKNAL_NSCHEDS << 1)
  57
  58#define SOCKNAL_PEER_HASH_SIZE  101   /* # peer lists */
  59#define SOCKNAL_RESCHED         100   /* # scheduler loops before reschedule */
  60#define SOCKNAL_INSANITY_RECONN 5000  /* connd is trying on reconn infinitely */
  61#define SOCKNAL_ENOMEM_RETRY    CFS_TICK /* jiffies between retries */
  62
  63#define SOCKNAL_SINGLE_FRAG_TX  0     /* disable multi-fragment sends */
  64#define SOCKNAL_SINGLE_FRAG_RX  0     /* disable multi-fragment receives */
  65
  66#define SOCKNAL_VERSION_DEBUG   0     /* enable protocol version debugging */
  67
  68/*
  69 * risk kmap deadlock on multi-frag I/O (backs off to single-frag if disabled).
  70 * no risk if we're not running on a CONFIG_HIGHMEM platform.
  71 */
  72#ifdef CONFIG_HIGHMEM
  73# define SOCKNAL_RISK_KMAP_DEADLOCK  0
  74#else
  75# define SOCKNAL_RISK_KMAP_DEADLOCK  1
  76#endif
  77
  78struct ksock_sched_info;
  79
  80struct ksock_sched {                            /* per scheduler state */
  81        spinlock_t              kss_lock;       /* serialise */
  82        struct list_head        kss_rx_conns;   /* conn waiting to be read */
  83        struct list_head        kss_tx_conns;   /* conn waiting to be written */
  84        struct list_head        kss_zombie_noop_txs; /* zombie noop tx list */
  85        wait_queue_head_t       kss_waitq;      /* where scheduler sleeps */
  86        int                     kss_nconns;     /* # connections assigned to
  87                                                 * this scheduler */
  88        struct ksock_sched_info *kss_info;      /* owner of it */
  89};
  90
  91struct ksock_sched_info {
  92        int                     ksi_nthreads_max; /* max allowed threads */
  93        int                     ksi_nthreads;     /* number of threads */
  94        int                     ksi_cpt;          /* CPT id */
  95        struct ksock_sched      *ksi_scheds;      /* array of schedulers */
  96};
  97
  98#define KSOCK_CPT_SHIFT           16
  99#define KSOCK_THREAD_ID(cpt, sid) (((cpt) << KSOCK_CPT_SHIFT) | (sid))
 100#define KSOCK_THREAD_CPT(id)      ((id) >> KSOCK_CPT_SHIFT)
 101#define KSOCK_THREAD_SID(id)      ((id) & ((1UL << KSOCK_CPT_SHIFT) - 1))
 102
 103struct ksock_interface {                        /* in-use interface */
 104        __u32           ksni_ipaddr;            /* interface's IP address */
 105        __u32           ksni_netmask;           /* interface's network mask */
 106        int             ksni_nroutes;           /* # routes using (active) */
 107        int             ksni_npeers;            /* # peers using (passive) */
 108        char            ksni_name[IFNAMSIZ];    /* interface name */
 109};
 110
 111struct ksock_tunables {
 112        int          *ksnd_timeout;            /* "stuck" socket timeout
 113                                                * (seconds) */
 114        int          *ksnd_nscheds;            /* # scheduler threads in each
 115                                                * pool while starting */
 116        int          *ksnd_nconnds;            /* # connection daemons */
 117        int          *ksnd_nconnds_max;        /* max # connection daemons */
 118        int          *ksnd_min_reconnectms;    /* first connection retry after
 119                                                * (ms)... */
 120        int          *ksnd_max_reconnectms;    /* ...exponentially increasing to
 121                                                * this */
 122        int          *ksnd_eager_ack;          /* make TCP ack eagerly? */
 123        int          *ksnd_typed_conns;        /* drive sockets by type? */
 124        int          *ksnd_min_bulk;           /* smallest "large" message */
 125        int          *ksnd_tx_buffer_size;     /* socket tx buffer size */
 126        int          *ksnd_rx_buffer_size;     /* socket rx buffer size */
 127        int          *ksnd_nagle;              /* enable NAGLE? */
 128        int          *ksnd_round_robin;        /* round robin for multiple
 129                                                * interfaces */
 130        int          *ksnd_keepalive;          /* # secs for sending keepalive
 131                                                * NOOP */
 132        int          *ksnd_keepalive_idle;     /* # idle secs before 1st probe
 133                                                */
 134        int          *ksnd_keepalive_count;    /* # probes */
 135        int          *ksnd_keepalive_intvl;    /* time between probes */
 136        int          *ksnd_credits;            /* # concurrent sends */
 137        int          *ksnd_peertxcredits;      /* # concurrent sends to 1 peer
 138                                                */
 139        int          *ksnd_peerrtrcredits;     /* # per-peer router buffer
 140                                                * credits */
 141        int          *ksnd_peertimeout;        /* seconds to consider peer dead
 142                                                */
 143        int          *ksnd_enable_csum;        /* enable check sum */
 144        int          *ksnd_inject_csum_error;  /* set non-zero to inject
 145                                                * checksum error */
 146        int          *ksnd_nonblk_zcack;       /* always send zc-ack on
 147                                                * non-blocking connection */
 148        unsigned int *ksnd_zc_min_payload;     /* minimum zero copy payload
 149                                                * size */
 150        int          *ksnd_zc_recv;            /* enable ZC receive (for
 151                                                * Chelsio TOE) */
 152        int          *ksnd_zc_recv_min_nfrags; /* minimum # of fragments to
 153                                                * enable ZC receive */
 154};
 155
 156struct ksock_net {
 157        __u64             ksnn_incarnation;     /* my epoch */
 158        spinlock_t        ksnn_lock;            /* serialise */
 159        struct list_head          ksnn_list;            /* chain on global list */
 160        int               ksnn_npeers;          /* # peers */
 161        int               ksnn_shutdown;        /* shutting down? */
 162        int               ksnn_ninterfaces;     /* IP interfaces */
 163        struct ksock_interface ksnn_interfaces[LNET_MAX_INTERFACES];
 164};
 165
 166/** connd timeout */
 167#define SOCKNAL_CONND_TIMEOUT  120
 168/** reserved thread for accepting & creating new connd */
 169#define SOCKNAL_CONND_RESV     1
 170
 171struct ksock_nal_data {
 172        int                     ksnd_init;              /* initialisation state
 173                                                         */
 174        int                     ksnd_nnets;             /* # networks set up */
 175        struct list_head        ksnd_nets;              /* list of nets */
 176        rwlock_t                ksnd_global_lock;       /* stabilize peer/conn
 177                                                         * ops */
 178        struct list_head        *ksnd_peers;            /* hash table of all my
 179                                                         * known peers */
 180        int                     ksnd_peer_hash_size;    /* size of ksnd_peers */
 181
 182        int                     ksnd_nthreads;          /* # live threads */
 183        int                     ksnd_shuttingdown;      /* tell threads to exit
 184                                                         */
 185        struct ksock_sched_info **ksnd_sched_info;      /* schedulers info */
 186
 187        atomic_t                ksnd_nactive_txs;       /* #active txs */
 188
 189        struct list_head        ksnd_deathrow_conns;    /* conns to close:
 190                                                         * reaper_lock*/
 191        struct list_head        ksnd_zombie_conns;      /* conns to free:
 192                                                         * reaper_lock */
 193        struct list_head        ksnd_enomem_conns;      /* conns to retry:
 194                                                         * reaper_lock*/
 195        wait_queue_head_t       ksnd_reaper_waitq;      /* reaper sleeps here */
 196        unsigned long           ksnd_reaper_waketime;   /* when reaper will wake
 197                                                         */
 198        spinlock_t              ksnd_reaper_lock;       /* serialise */
 199
 200        int                     ksnd_enomem_tx;         /* test ENOMEM sender */
 201        int                     ksnd_stall_tx;          /* test sluggish sender
 202                                                         */
 203        int                     ksnd_stall_rx;          /* test sluggish
 204                                                         * receiver */
 205
 206        struct list_head        ksnd_connd_connreqs;    /* incoming connection
 207                                                         * requests */
 208        struct list_head        ksnd_connd_routes;      /* routes waiting to be
 209                                                         * connected */
 210        wait_queue_head_t       ksnd_connd_waitq;       /* connds sleep here */
 211        int                     ksnd_connd_connecting;  /* # connds connecting
 212                                                         */
 213        time64_t                ksnd_connd_failed_stamp;/* time stamp of the
 214                                                         * last failed
 215                                                         * connecting attempt */
 216        time64_t                ksnd_connd_starting_stamp;/* time stamp of the
 217                                                           * last starting connd
 218                                                           */
 219        unsigned                ksnd_connd_starting;    /* # starting connd */
 220        unsigned                ksnd_connd_running;     /* # running connd */
 221        spinlock_t              ksnd_connd_lock;        /* serialise */
 222
 223        struct list_head        ksnd_idle_noop_txs;     /* list head for freed
 224                                                         * noop tx */
 225        spinlock_t              ksnd_tx_lock;           /* serialise, g_lock
 226                                                         * unsafe */
 227
 228};
 229
 230#define SOCKNAL_INIT_NOTHING 0
 231#define SOCKNAL_INIT_DATA    1
 232#define SOCKNAL_INIT_ALL     2
 233
 234/*
 235 * A packet just assembled for transmission is represented by 1 or more
 236 * struct iovec fragments (the first frag contains the portals header),
 237 * followed by 0 or more lnet_kiov_t fragments.
 238 *
 239 * On the receive side, initially 1 struct iovec fragment is posted for
 240 * receive (the header).  Once the header has been received, the payload is
 241 * received into either struct iovec or lnet_kiov_t fragments, depending on
 242 * what the header matched or whether the message needs forwarding.
 243 */
 244struct ksock_conn;  /* forward ref */
 245struct ksock_peer;  /* forward ref */
 246struct ksock_route; /* forward ref */
 247struct ksock_proto; /* forward ref */
 248
 249struct ksock_tx {                          /* transmit packet */
 250        struct list_head  tx_list;         /* queue on conn for transmission etc
 251                                            */
 252        struct list_head  tx_zc_list;      /* queue on peer for ZC request */
 253        atomic_t          tx_refcount;     /* tx reference count */
 254        int               tx_nob;          /* # packet bytes */
 255        int               tx_resid;        /* residual bytes */
 256        int               tx_niov;         /* # packet iovec frags */
 257        struct kvec       *tx_iov;         /* packet iovec frags */
 258        int               tx_nkiov;        /* # packet page frags */
 259        unsigned short    tx_zc_aborted;   /* aborted ZC request */
 260        unsigned short    tx_zc_capable:1; /* payload is large enough for ZC */
 261        unsigned short    tx_zc_checked:1; /* Have I checked if I should ZC? */
 262        unsigned short    tx_nonblk:1;     /* it's a non-blocking ACK */
 263        lnet_kiov_t       *tx_kiov;        /* packet page frags */
 264        struct ksock_conn *tx_conn;        /* owning conn */
 265        lnet_msg_t        *tx_lnetmsg;     /* lnet message for lnet_finalize()
 266                                            */
 267        unsigned long     tx_deadline;     /* when (in jiffies) tx times out */
 268        ksock_msg_t       tx_msg;          /* socklnd message buffer */
 269        int               tx_desc_size;    /* size of this descriptor */
 270        union {
 271                struct {
 272                        struct kvec iov;     /* virt hdr */
 273                        lnet_kiov_t kiov[0]; /* paged payload */
 274                } paged;
 275                struct {
 276                        struct kvec iov[1];  /* virt hdr + payload */
 277                } virt;
 278        } tx_frags;
 279};
 280
 281#define KSOCK_NOOP_TX_SIZE (offsetof(struct ksock_tx, tx_frags.paged.kiov[0]))
 282
 283/* network zero copy callback descriptor embedded in struct ksock_tx */
 284
 285/*
 286 * space for the rx frag descriptors; we either read a single contiguous
 287 * header, or up to LNET_MAX_IOV frags of payload of either type.
 288 */
 289union ksock_rxiovspace {
 290        struct kvec      iov[LNET_MAX_IOV];
 291        lnet_kiov_t      kiov[LNET_MAX_IOV];
 292};
 293
 294#define SOCKNAL_RX_KSM_HEADER   1 /* reading ksock message header */
 295#define SOCKNAL_RX_LNET_HEADER  2 /* reading lnet message header */
 296#define SOCKNAL_RX_PARSE        3 /* Calling lnet_parse() */
 297#define SOCKNAL_RX_PARSE_WAIT   4 /* waiting to be told to read the body */
 298#define SOCKNAL_RX_LNET_PAYLOAD 5 /* reading lnet payload (to deliver here) */
 299#define SOCKNAL_RX_SLOP         6 /* skipping body */
 300
 301struct ksock_conn {
 302        struct ksock_peer  *ksnc_peer;        /* owning peer */
 303        struct ksock_route *ksnc_route;       /* owning route */
 304        struct list_head   ksnc_list;         /* stash on peer's conn list */
 305        struct socket      *ksnc_sock;        /* actual socket */
 306        void               *ksnc_saved_data_ready;  /* socket's original
 307                                                     * data_ready() callback */
 308        void               *ksnc_saved_write_space; /* socket's original
 309                                                     * write_space() callback */
 310        atomic_t           ksnc_conn_refcount;/* conn refcount */
 311        atomic_t           ksnc_sock_refcount;/* sock refcount */
 312        struct ksock_sched *ksnc_scheduler;     /* who schedules this connection
 313                                                 */
 314        __u32              ksnc_myipaddr;     /* my IP */
 315        __u32              ksnc_ipaddr;       /* peer's IP */
 316        int                ksnc_port;         /* peer's port */
 317        signed int         ksnc_type:3;       /* type of connection, should be
 318                                               * signed value */
 319        unsigned int       ksnc_closing:1;    /* being shut down */
 320        unsigned int       ksnc_flip:1;       /* flip or not, only for V2.x */
 321        unsigned int       ksnc_zc_capable:1; /* enable to ZC */
 322        struct ksock_proto *ksnc_proto;       /* protocol for the connection */
 323
 324        /* reader */
 325        struct list_head   ksnc_rx_list;      /* where I enq waiting input or a
 326                                               * forwarding descriptor */
 327        unsigned long      ksnc_rx_deadline;  /* when (in jiffies) receive times
 328                                               * out */
 329        __u8               ksnc_rx_started;   /* started receiving a message */
 330        __u8               ksnc_rx_ready;     /* data ready to read */
 331        __u8               ksnc_rx_scheduled; /* being progressed */
 332        __u8               ksnc_rx_state;     /* what is being read */
 333        int                ksnc_rx_nob_left;  /* # bytes to next hdr/body */
 334        int                ksnc_rx_nob_wanted;/* bytes actually wanted */
 335        int                ksnc_rx_niov;      /* # iovec frags */
 336        struct kvec        *ksnc_rx_iov;      /* the iovec frags */
 337        int                ksnc_rx_nkiov;     /* # page frags */
 338        lnet_kiov_t        *ksnc_rx_kiov;     /* the page frags */
 339        union ksock_rxiovspace ksnc_rx_iov_space; /* space for frag descriptors */
 340        __u32              ksnc_rx_csum;      /* partial checksum for incoming
 341                                               * data */
 342        void               *ksnc_cookie;      /* rx lnet_finalize passthru arg
 343                                               */
 344        ksock_msg_t        ksnc_msg;          /* incoming message buffer:
 345                                               * V2.x message takes the
 346                                               * whole struct
 347                                               * V1.x message is a bare
 348                                               * lnet_hdr_t, it's stored in
 349                                               * ksnc_msg.ksm_u.lnetmsg */
 350
 351        /* WRITER */
 352        struct list_head   ksnc_tx_list;      /* where I enq waiting for output
 353                                               * space */
 354        struct list_head   ksnc_tx_queue;     /* packets waiting to be sent */
 355        struct ksock_tx         *ksnc_tx_carrier;  /* next TX that can carry a LNet
 356                                               * message or ZC-ACK */
 357        unsigned long      ksnc_tx_deadline;  /* when (in jiffies) tx times out
 358                                               */
 359        int                ksnc_tx_bufnob;    /* send buffer marker */
 360        atomic_t           ksnc_tx_nob;       /* # bytes queued */
 361        int                ksnc_tx_ready;     /* write space */
 362        int                ksnc_tx_scheduled; /* being progressed */
 363        unsigned long      ksnc_tx_last_post; /* time stamp of the last posted
 364                                               * TX */
 365};
 366
 367struct ksock_route {
 368        struct list_head  ksnr_list;           /* chain on peer route list */
 369        struct list_head  ksnr_connd_list;     /* chain on ksnr_connd_routes */
 370        struct ksock_peer *ksnr_peer;          /* owning peer */
 371        atomic_t          ksnr_refcount;       /* # users */
 372        unsigned long     ksnr_timeout;        /* when (in jiffies) reconnection
 373                                                * can happen next */
 374        long              ksnr_retry_interval; /* how long between retries */
 375        __u32             ksnr_myipaddr;       /* my IP */
 376        __u32             ksnr_ipaddr;         /* IP address to connect to */
 377        int               ksnr_port;           /* port to connect to */
 378        unsigned int      ksnr_scheduled:1;    /* scheduled for attention */
 379        unsigned int      ksnr_connecting:1;   /* connection establishment in
 380                                                * progress */
 381        unsigned int      ksnr_connected:4;    /* connections established by
 382                                                * type */
 383        unsigned int      ksnr_deleted:1;      /* been removed from peer? */
 384        unsigned int      ksnr_share_count;    /* created explicitly? */
 385        int               ksnr_conn_count;     /* # conns established by this
 386                                                * route */
 387};
 388
 389#define SOCKNAL_KEEPALIVE_PING 1 /* cookie for keepalive ping */
 390
 391struct ksock_peer {
 392        struct list_head   ksnp_list;           /* stash on global peer list */
 393        unsigned long      ksnp_last_alive;     /* when (in jiffies) I was last
 394                                                 * alive */
 395        lnet_process_id_t  ksnp_id;             /* who's on the other end(s) */
 396        atomic_t           ksnp_refcount;       /* # users */
 397        int                ksnp_sharecount;     /* lconf usage counter */
 398        int                ksnp_closing;        /* being closed */
 399        int                ksnp_accepting;      /* # passive connections pending
 400                                                 */
 401        int                ksnp_error;          /* errno on closing last conn */
 402        __u64              ksnp_zc_next_cookie; /* ZC completion cookie */
 403        __u64              ksnp_incarnation;    /* latest known peer incarnation
 404                                                 */
 405        struct ksock_proto *ksnp_proto;         /* latest known peer protocol */
 406        struct list_head   ksnp_conns;          /* all active connections */
 407        struct list_head   ksnp_routes;         /* routes */
 408        struct list_head   ksnp_tx_queue;       /* waiting packets */
 409        spinlock_t         ksnp_lock;           /* serialize, g_lock unsafe */
 410        struct list_head   ksnp_zc_req_list;    /* zero copy requests wait for
 411                                                 * ACK  */
 412        unsigned long      ksnp_send_keepalive; /* time to send keepalive */
 413        lnet_ni_t          *ksnp_ni;            /* which network */
 414        int                ksnp_n_passive_ips;  /* # of... */
 415
 416        /* preferred local interfaces */
 417        __u32              ksnp_passive_ips[LNET_MAX_INTERFACES];
 418};
 419
 420struct ksock_connreq {
 421        struct list_head ksncr_list;  /* stash on ksnd_connd_connreqs */
 422        lnet_ni_t        *ksncr_ni;   /* chosen NI */
 423        struct socket    *ksncr_sock; /* accepted socket */
 424};
 425
 426extern struct ksock_nal_data ksocknal_data;
 427extern struct ksock_tunables ksocknal_tunables;
 428
 429#define SOCKNAL_MATCH_NO  0 /* TX can't match type of connection */
 430#define SOCKNAL_MATCH_YES 1 /* TX matches type of connection */
 431#define SOCKNAL_MATCH_MAY 2 /* TX can be sent on the connection, but not
 432                             * preferred */
 433
 434struct ksock_proto {
 435        /* version number of protocol */
 436        int        pro_version;
 437
 438        /* handshake function */
 439        int        (*pro_send_hello)(struct ksock_conn *, ksock_hello_msg_t *);
 440
 441        /* handshake function */
 442        int        (*pro_recv_hello)(struct ksock_conn *, ksock_hello_msg_t *, int);
 443
 444        /* message pack */
 445        void       (*pro_pack)(struct ksock_tx *);
 446
 447        /* message unpack */
 448        void       (*pro_unpack)(ksock_msg_t *);
 449
 450        /* queue tx on the connection */
 451        struct ksock_tx *(*pro_queue_tx_msg)(struct ksock_conn *, struct ksock_tx *);
 452
 453        /* queue ZC ack on the connection */
 454        int        (*pro_queue_tx_zcack)(struct ksock_conn *, struct ksock_tx *, __u64);
 455
 456        /* handle ZC request */
 457        int        (*pro_handle_zcreq)(struct ksock_conn *, __u64, int);
 458
 459        /* handle ZC ACK */
 460        int        (*pro_handle_zcack)(struct ksock_conn *, __u64, __u64);
 461
 462        /*
 463         * msg type matches the connection type:
 464         * return value:
 465         *   return MATCH_NO  : no
 466         *   return MATCH_YES : matching type
 467         *   return MATCH_MAY : can be backup
 468         */
 469        int        (*pro_match_tx)(struct ksock_conn *, struct ksock_tx *, int);
 470};
 471
 472extern struct ksock_proto ksocknal_protocol_v1x;
 473extern struct ksock_proto ksocknal_protocol_v2x;
 474extern struct ksock_proto ksocknal_protocol_v3x;
 475
 476#define KSOCK_PROTO_V1_MAJOR LNET_PROTO_TCP_VERSION_MAJOR
 477#define KSOCK_PROTO_V1_MINOR LNET_PROTO_TCP_VERSION_MINOR
 478#define KSOCK_PROTO_V1       KSOCK_PROTO_V1_MAJOR
 479
 480#ifndef CPU_MASK_NONE
 481#define CPU_MASK_NONE   0UL
 482#endif
 483
 484static inline __u32 ksocknal_csum(__u32 crc, unsigned char const *p, size_t len)
 485{
 486#if 1
 487        return crc32_le(crc, p, len);
 488#else
 489        while (len-- > 0)
 490                crc = ((crc + 0x100) & ~0xff) | ((crc + *p++) & 0xff) ;
 491        return crc;
 492#endif
 493}
 494
 495static inline int
 496ksocknal_route_mask(void)
 497{
 498        if (!*ksocknal_tunables.ksnd_typed_conns)
 499                return (1 << SOCKLND_CONN_ANY);
 500
 501        return ((1 << SOCKLND_CONN_CONTROL) |
 502                (1 << SOCKLND_CONN_BULK_IN) |
 503                (1 << SOCKLND_CONN_BULK_OUT));
 504}
 505
 506static inline struct list_head *
 507ksocknal_nid2peerlist(lnet_nid_t nid)
 508{
 509        unsigned int hash = ((unsigned int)nid) % ksocknal_data.ksnd_peer_hash_size;
 510
 511        return &ksocknal_data.ksnd_peers[hash];
 512}
 513
 514static inline void
 515ksocknal_conn_addref(struct ksock_conn *conn)
 516{
 517        LASSERT(atomic_read(&conn->ksnc_conn_refcount) > 0);
 518        atomic_inc(&conn->ksnc_conn_refcount);
 519}
 520
 521void ksocknal_queue_zombie_conn(struct ksock_conn *conn);
 522void ksocknal_finalize_zcreq(struct ksock_conn *conn);
 523
 524static inline void
 525ksocknal_conn_decref(struct ksock_conn *conn)
 526{
 527        LASSERT(atomic_read(&conn->ksnc_conn_refcount) > 0);
 528        if (atomic_dec_and_test(&conn->ksnc_conn_refcount))
 529                ksocknal_queue_zombie_conn(conn);
 530}
 531
 532static inline int
 533ksocknal_connsock_addref(struct ksock_conn *conn)
 534{
 535        int rc = -ESHUTDOWN;
 536
 537        read_lock(&ksocknal_data.ksnd_global_lock);
 538        if (!conn->ksnc_closing) {
 539                LASSERT(atomic_read(&conn->ksnc_sock_refcount) > 0);
 540                atomic_inc(&conn->ksnc_sock_refcount);
 541                rc = 0;
 542        }
 543        read_unlock(&ksocknal_data.ksnd_global_lock);
 544
 545        return rc;
 546}
 547
 548static inline void
 549ksocknal_connsock_decref(struct ksock_conn *conn)
 550{
 551        LASSERT(atomic_read(&conn->ksnc_sock_refcount) > 0);
 552        if (atomic_dec_and_test(&conn->ksnc_sock_refcount)) {
 553                LASSERT(conn->ksnc_closing);
 554                sock_release(conn->ksnc_sock);
 555                conn->ksnc_sock = NULL;
 556                ksocknal_finalize_zcreq(conn);
 557        }
 558}
 559
 560static inline void
 561ksocknal_tx_addref(struct ksock_tx *tx)
 562{
 563        LASSERT(atomic_read(&tx->tx_refcount) > 0);
 564        atomic_inc(&tx->tx_refcount);
 565}
 566
 567void ksocknal_tx_prep(struct ksock_conn *, struct ksock_tx *tx);
 568void ksocknal_tx_done(lnet_ni_t *ni, struct ksock_tx *tx);
 569
 570static inline void
 571ksocknal_tx_decref(struct ksock_tx *tx)
 572{
 573        LASSERT(atomic_read(&tx->tx_refcount) > 0);
 574        if (atomic_dec_and_test(&tx->tx_refcount))
 575                ksocknal_tx_done(NULL, tx);
 576}
 577
 578static inline void
 579ksocknal_route_addref(struct ksock_route *route)
 580{
 581        LASSERT(atomic_read(&route->ksnr_refcount) > 0);
 582        atomic_inc(&route->ksnr_refcount);
 583}
 584
 585void ksocknal_destroy_route(struct ksock_route *route);
 586
 587static inline void
 588ksocknal_route_decref(struct ksock_route *route)
 589{
 590        LASSERT(atomic_read(&route->ksnr_refcount) > 0);
 591        if (atomic_dec_and_test(&route->ksnr_refcount))
 592                ksocknal_destroy_route(route);
 593}
 594
 595static inline void
 596ksocknal_peer_addref(struct ksock_peer *peer)
 597{
 598        LASSERT(atomic_read(&peer->ksnp_refcount) > 0);
 599        atomic_inc(&peer->ksnp_refcount);
 600}
 601
 602void ksocknal_destroy_peer(struct ksock_peer *peer);
 603
 604static inline void
 605ksocknal_peer_decref(struct ksock_peer *peer)
 606{
 607        LASSERT(atomic_read(&peer->ksnp_refcount) > 0);
 608        if (atomic_dec_and_test(&peer->ksnp_refcount))
 609                ksocknal_destroy_peer(peer);
 610}
 611
 612int ksocknal_startup(lnet_ni_t *ni);
 613void ksocknal_shutdown(lnet_ni_t *ni);
 614int ksocknal_ctl(lnet_ni_t *ni, unsigned int cmd, void *arg);
 615int ksocknal_send(lnet_ni_t *ni, void *private, lnet_msg_t *lntmsg);
 616int ksocknal_recv(lnet_ni_t *ni, void *private, lnet_msg_t *lntmsg,
 617                  int delayed, struct iov_iter *to, unsigned int rlen);
 618int ksocknal_accept(lnet_ni_t *ni, struct socket *sock);
 619
 620int ksocknal_add_peer(lnet_ni_t *ni, lnet_process_id_t id, __u32 ip, int port);
 621struct ksock_peer *ksocknal_find_peer_locked(lnet_ni_t *ni, lnet_process_id_t id);
 622struct ksock_peer *ksocknal_find_peer(lnet_ni_t *ni, lnet_process_id_t id);
 623void ksocknal_peer_failed(struct ksock_peer *peer);
 624int ksocknal_create_conn(lnet_ni_t *ni, struct ksock_route *route,
 625                         struct socket *sock, int type);
 626void ksocknal_close_conn_locked(struct ksock_conn *conn, int why);
 627void ksocknal_terminate_conn(struct ksock_conn *conn);
 628void ksocknal_destroy_conn(struct ksock_conn *conn);
 629int  ksocknal_close_peer_conns_locked(struct ksock_peer *peer,
 630                                      __u32 ipaddr, int why);
 631int ksocknal_close_conn_and_siblings(struct ksock_conn *conn, int why);
 632int ksocknal_close_matching_conns(lnet_process_id_t id, __u32 ipaddr);
 633struct ksock_conn *ksocknal_find_conn_locked(struct ksock_peer *peer,
 634                                             struct ksock_tx *tx, int nonblk);
 635
 636int  ksocknal_launch_packet(lnet_ni_t *ni, struct ksock_tx *tx,
 637                            lnet_process_id_t id);
 638struct ksock_tx *ksocknal_alloc_tx(int type, int size);
 639void ksocknal_free_tx(struct ksock_tx *tx);
 640struct ksock_tx *ksocknal_alloc_tx_noop(__u64 cookie, int nonblk);
 641void ksocknal_next_tx_carrier(struct ksock_conn *conn);
 642void ksocknal_queue_tx_locked(struct ksock_tx *tx, struct ksock_conn *conn);
 643void ksocknal_txlist_done(lnet_ni_t *ni, struct list_head *txlist, int error);
 644void ksocknal_notify(lnet_ni_t *ni, lnet_nid_t gw_nid, int alive);
 645void ksocknal_query(struct lnet_ni *ni, lnet_nid_t nid, unsigned long *when);
 646int ksocknal_thread_start(int (*fn)(void *arg), void *arg, char *name);
 647void ksocknal_thread_fini(void);
 648void ksocknal_launch_all_connections_locked(struct ksock_peer *peer);
 649struct ksock_route *ksocknal_find_connectable_route_locked(struct ksock_peer *peer);
 650struct ksock_route *ksocknal_find_connecting_route_locked(struct ksock_peer *peer);
 651int ksocknal_new_packet(struct ksock_conn *conn, int skip);
 652int ksocknal_scheduler(void *arg);
 653int ksocknal_connd(void *arg);
 654int ksocknal_reaper(void *arg);
 655int ksocknal_send_hello(lnet_ni_t *ni, struct ksock_conn *conn,
 656                        lnet_nid_t peer_nid, ksock_hello_msg_t *hello);
 657int ksocknal_recv_hello(lnet_ni_t *ni, struct ksock_conn *conn,
 658                        ksock_hello_msg_t *hello, lnet_process_id_t *id,
 659                        __u64 *incarnation);
 660void ksocknal_read_callback(struct ksock_conn *conn);
 661void ksocknal_write_callback(struct ksock_conn *conn);
 662
 663int ksocknal_lib_zc_capable(struct ksock_conn *conn);
 664void ksocknal_lib_save_callback(struct socket *sock, struct ksock_conn *conn);
 665void ksocknal_lib_set_callback(struct socket *sock,  struct ksock_conn *conn);
 666void ksocknal_lib_reset_callback(struct socket *sock, struct ksock_conn *conn);
 667void ksocknal_lib_push_conn(struct ksock_conn *conn);
 668int ksocknal_lib_get_conn_addrs(struct ksock_conn *conn);
 669int ksocknal_lib_setup_sock(struct socket *so);
 670int ksocknal_lib_send_iov(struct ksock_conn *conn, struct ksock_tx *tx);
 671int ksocknal_lib_send_kiov(struct ksock_conn *conn, struct ksock_tx *tx);
 672void ksocknal_lib_eager_ack(struct ksock_conn *conn);
 673int ksocknal_lib_recv_iov(struct ksock_conn *conn);
 674int ksocknal_lib_recv_kiov(struct ksock_conn *conn);
 675int ksocknal_lib_get_conn_tunables(struct ksock_conn *conn, int *txmem,
 676                                   int *rxmem, int *nagle);
 677
 678void ksocknal_read_callback(struct ksock_conn *conn);
 679void ksocknal_write_callback(struct ksock_conn *conn);
 680
 681int ksocknal_tunables_init(void);
 682
 683void ksocknal_lib_csum_tx(struct ksock_tx *tx);
 684
 685int ksocknal_lib_memory_pressure(struct ksock_conn *conn);
 686int ksocknal_lib_bind_thread_to_cpu(int id);
 687
 688#endif /* _SOCKLND_SOCKLND_H_ */
 689