1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Kernel Connection Multiplexor 4 * 5 * Copyright (c) 2016 Tom Herbert <tom@herbertland.com> 6 */ 7 8#ifndef __NET_KCM_H_ 9#define __NET_KCM_H_ 10 11#include <linux/skbuff.h> 12#include <net/sock.h> 13#include <net/strparser.h> 14#include <uapi/linux/kcm.h> 15 16extern unsigned int kcm_net_id; 17 18#define KCM_STATS_ADD(stat, count) ((stat) += (count)) 19#define KCM_STATS_INCR(stat) ((stat)++) 20 21struct kcm_psock_stats { 22 unsigned long long tx_msgs; 23 unsigned long long tx_bytes; 24 unsigned long long reserved; 25 unsigned long long unreserved; 26 unsigned int tx_aborts; 27}; 28 29struct kcm_mux_stats { 30 unsigned long long rx_msgs; 31 unsigned long long rx_bytes; 32 unsigned long long tx_msgs; 33 unsigned long long tx_bytes; 34 unsigned int rx_ready_drops; 35 unsigned int tx_retries; 36 unsigned int psock_attach; 37 unsigned int psock_unattach_rsvd; 38 unsigned int psock_unattach; 39}; 40 41struct kcm_stats { 42 unsigned long long rx_msgs; 43 unsigned long long rx_bytes; 44 unsigned long long tx_msgs; 45 unsigned long long tx_bytes; 46}; 47 48struct kcm_tx_msg { 49 unsigned int sent; 50 unsigned int fragidx; 51 unsigned int frag_offset; 52 unsigned int msg_flags; 53 struct sk_buff *frag_skb; 54 struct sk_buff *last_skb; 55}; 56 57/* Socket structure for KCM client sockets */ 58struct kcm_sock { 59 struct sock sk; 60 struct kcm_mux *mux; 61 struct list_head kcm_sock_list; 62 int index; 63 u32 done : 1; 64 struct work_struct done_work; 65 66 struct kcm_stats stats; 67 68 /* Transmit */ 69 struct kcm_psock *tx_psock; 70 struct work_struct tx_work; 71 struct list_head wait_psock_list; 72 struct sk_buff *seq_skb; 73 u32 tx_stopped : 1; 74 75 /* Don't use bit fields here, these are set under different locks */ 76 bool tx_wait; 77 bool tx_wait_more; 78 79 /* Receive */ 80 struct kcm_psock *rx_psock; 81 struct list_head wait_rx_list; /* KCMs waiting for receiving */ 82 bool rx_wait; 83 u32 rx_disabled : 1; 84}; 85 86struct bpf_prog; 87 88/* Structure for an attached lower socket */ 89struct kcm_psock { 90 struct sock *sk; 91 struct strparser strp; 92 struct kcm_mux *mux; 93 int index; 94 95 u32 tx_stopped : 1; 96 u32 done : 1; 97 u32 unattaching : 1; 98 99 void (*save_state_change)(struct sock *sk); 100 void (*save_data_ready)(struct sock *sk); 101 void (*save_write_space)(struct sock *sk); 102 103 struct list_head psock_list; 104 105 struct kcm_psock_stats stats; 106 107 /* Receive */ 108 struct list_head psock_ready_list; 109 struct bpf_prog *bpf_prog; 110 struct kcm_sock *rx_kcm; 111 unsigned long long saved_rx_bytes; 112 unsigned long long saved_rx_msgs; 113 struct sk_buff *ready_rx_msg; 114 115 /* Transmit */ 116 struct kcm_sock *tx_kcm; 117 struct list_head psock_avail_list; 118 unsigned long long saved_tx_bytes; 119 unsigned long long saved_tx_msgs; 120}; 121 122/* Per net MUX list */ 123struct kcm_net { 124 struct mutex mutex; 125 struct kcm_psock_stats aggregate_psock_stats; 126 struct kcm_mux_stats aggregate_mux_stats; 127 struct strp_aggr_stats aggregate_strp_stats; 128 struct list_head mux_list; 129 int count; 130}; 131 132/* Structure for a MUX */ 133struct kcm_mux { 134 struct list_head kcm_mux_list; 135 struct rcu_head rcu; 136 struct kcm_net *knet; 137 138 struct list_head kcm_socks; /* All KCM sockets on MUX */ 139 int kcm_socks_cnt; /* Total KCM socket count for MUX */ 140 struct list_head psocks; /* List of all psocks on MUX */ 141 int psocks_cnt; /* Total attached sockets */ 142 143 struct kcm_mux_stats stats; 144 struct kcm_psock_stats aggregate_psock_stats; 145 struct strp_aggr_stats aggregate_strp_stats; 146 147 /* Receive */ 148 spinlock_t rx_lock ____cacheline_aligned_in_smp; 149 struct list_head kcm_rx_waiters; /* KCMs waiting for receiving */ 150 struct list_head psocks_ready; /* List of psocks with a msg ready */ 151 struct sk_buff_head rx_hold_queue; 152 153 /* Transmit */ 154 spinlock_t lock ____cacheline_aligned_in_smp; /* TX and mux locking */ 155 struct list_head psocks_avail; /* List of available psocks */ 156 struct list_head kcm_tx_waiters; /* KCMs waiting for a TX psock */ 157}; 158 159#ifdef CONFIG_PROC_FS 160int kcm_proc_init(void); 161void kcm_proc_exit(void); 162#else 163static inline int kcm_proc_init(void) { return 0; } 164static inline void kcm_proc_exit(void) { } 165#endif 166 167static inline void aggregate_psock_stats(struct kcm_psock_stats *stats, 168 struct kcm_psock_stats *agg_stats) 169{ 170 /* Save psock statistics in the mux when psock is being unattached. */ 171 172#define SAVE_PSOCK_STATS(_stat) (agg_stats->_stat += stats->_stat) 173 SAVE_PSOCK_STATS(tx_msgs); 174 SAVE_PSOCK_STATS(tx_bytes); 175 SAVE_PSOCK_STATS(reserved); 176 SAVE_PSOCK_STATS(unreserved); 177 SAVE_PSOCK_STATS(tx_aborts); 178#undef SAVE_PSOCK_STATS 179} 180 181static inline void aggregate_mux_stats(struct kcm_mux_stats *stats, 182 struct kcm_mux_stats *agg_stats) 183{ 184 /* Save psock statistics in the mux when psock is being unattached. */ 185 186#define SAVE_MUX_STATS(_stat) (agg_stats->_stat += stats->_stat) 187 SAVE_MUX_STATS(rx_msgs); 188 SAVE_MUX_STATS(rx_bytes); 189 SAVE_MUX_STATS(tx_msgs); 190 SAVE_MUX_STATS(tx_bytes); 191 SAVE_MUX_STATS(rx_ready_drops); 192 SAVE_MUX_STATS(psock_attach); 193 SAVE_MUX_STATS(psock_unattach_rsvd); 194 SAVE_MUX_STATS(psock_unattach); 195#undef SAVE_MUX_STATS 196} 197 198#endif /* __NET_KCM_H_ */ 199