1/* 2 * Kernel Connection Multiplexor 3 * 4 * Copyright (c) 2016 Tom Herbert <tom@herbertland.com> 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 8 * as published by the Free Software Foundation. 9 */ 10 11#ifndef __NET_KCM_H_ 12#define __NET_KCM_H_ 13 14#include <linux/skbuff.h> 15#include <net/sock.h> 16#include <uapi/linux/kcm.h> 17 18extern unsigned int kcm_net_id; 19 20#define KCM_STATS_ADD(stat, count) ((stat) += (count)) 21#define KCM_STATS_INCR(stat) ((stat)++) 22 23struct kcm_psock_stats { 24 unsigned long long rx_msgs; 25 unsigned long long rx_bytes; 26 unsigned long long tx_msgs; 27 unsigned long long tx_bytes; 28 unsigned int rx_aborts; 29 unsigned int rx_mem_fail; 30 unsigned int rx_need_more_hdr; 31 unsigned int rx_msg_too_big; 32 unsigned int rx_msg_timeouts; 33 unsigned int rx_bad_hdr_len; 34 unsigned long long reserved; 35 unsigned long long unreserved; 36 unsigned int tx_aborts; 37}; 38 39struct kcm_mux_stats { 40 unsigned long long rx_msgs; 41 unsigned long long rx_bytes; 42 unsigned long long tx_msgs; 43 unsigned long long tx_bytes; 44 unsigned int rx_ready_drops; 45 unsigned int tx_retries; 46 unsigned int psock_attach; 47 unsigned int psock_unattach_rsvd; 48 unsigned int psock_unattach; 49}; 50 51struct kcm_stats { 52 unsigned long long rx_msgs; 53 unsigned long long rx_bytes; 54 unsigned long long tx_msgs; 55 unsigned long long tx_bytes; 56}; 57 58struct kcm_tx_msg { 59 unsigned int sent; 60 unsigned int fragidx; 61 unsigned int frag_offset; 62 unsigned int msg_flags; 63 struct sk_buff *frag_skb; 64 struct sk_buff *last_skb; 65}; 66 67struct kcm_rx_msg { 68 int full_len; 69 int accum_len; 70 int offset; 71 int early_eaten; 72}; 73 74/* Socket structure for KCM client sockets */ 75struct kcm_sock { 76 struct sock sk; 77 struct kcm_mux *mux; 78 struct list_head kcm_sock_list; 79 int index; 80 u32 done : 1; 81 struct work_struct done_work; 82 83 struct kcm_stats stats; 84 85 /* Transmit */ 86 struct kcm_psock *tx_psock; 87 struct work_struct tx_work; 88 struct list_head wait_psock_list; 89 struct sk_buff *seq_skb; 90 91 /* Don't use bit fields here, these are set under different locks */ 92 bool tx_wait; 93 bool tx_wait_more; 94 95 /* Receive */ 96 struct kcm_psock *rx_psock; 97 struct list_head wait_rx_list; /* KCMs waiting for receiving */ 98 bool rx_wait; 99 u32 rx_disabled : 1; 100}; 101 102struct bpf_prog; 103 104/* Structure for an attached lower socket */ 105struct kcm_psock { 106 struct sock *sk; 107 struct kcm_mux *mux; 108 int index; 109 110 u32 tx_stopped : 1; 111 u32 rx_stopped : 1; 112 u32 done : 1; 113 u32 unattaching : 1; 114 115 void (*save_state_change)(struct sock *sk); 116 void (*save_data_ready)(struct sock *sk); 117 void (*save_write_space)(struct sock *sk); 118 119 struct list_head psock_list; 120 121 struct kcm_psock_stats stats; 122 123 /* Receive */ 124 struct sk_buff *rx_skb_head; 125 struct sk_buff **rx_skb_nextp; 126 struct sk_buff *ready_rx_msg; 127 struct list_head psock_ready_list; 128 struct work_struct rx_work; 129 struct delayed_work rx_delayed_work; 130 struct bpf_prog *bpf_prog; 131 struct kcm_sock *rx_kcm; 132 unsigned long long saved_rx_bytes; 133 unsigned long long saved_rx_msgs; 134 struct timer_list rx_msg_timer; 135 unsigned int rx_need_bytes; 136 137 /* Transmit */ 138 struct kcm_sock *tx_kcm; 139 struct list_head psock_avail_list; 140 unsigned long long saved_tx_bytes; 141 unsigned long long saved_tx_msgs; 142}; 143 144/* Per net MUX list */ 145struct kcm_net { 146 struct mutex mutex; 147 struct kcm_psock_stats aggregate_psock_stats; 148 struct kcm_mux_stats aggregate_mux_stats; 149 struct list_head mux_list; 150 int count; 151}; 152 153/* Structure for a MUX */ 154struct kcm_mux { 155 struct list_head kcm_mux_list; 156 struct rcu_head rcu; 157 struct kcm_net *knet; 158 159 struct list_head kcm_socks; /* All KCM sockets on MUX */ 160 int kcm_socks_cnt; /* Total KCM socket count for MUX */ 161 struct list_head psocks; /* List of all psocks on MUX */ 162 int psocks_cnt; /* Total attached sockets */ 163 164 struct kcm_mux_stats stats; 165 struct kcm_psock_stats aggregate_psock_stats; 166 167 /* Receive */ 168 spinlock_t rx_lock ____cacheline_aligned_in_smp; 169 struct list_head kcm_rx_waiters; /* KCMs waiting for receiving */ 170 struct list_head psocks_ready; /* List of psocks with a msg ready */ 171 struct sk_buff_head rx_hold_queue; 172 173 /* Transmit */ 174 spinlock_t lock ____cacheline_aligned_in_smp; /* TX and mux locking */ 175 struct list_head psocks_avail; /* List of available psocks */ 176 struct list_head kcm_tx_waiters; /* KCMs waiting for a TX psock */ 177}; 178 179#ifdef CONFIG_PROC_FS 180int kcm_proc_init(void); 181void kcm_proc_exit(void); 182#else 183static inline int kcm_proc_init(void) { return 0; } 184static inline void kcm_proc_exit(void) { } 185#endif 186 187static inline void aggregate_psock_stats(struct kcm_psock_stats *stats, 188 struct kcm_psock_stats *agg_stats) 189{ 190 /* Save psock statistics in the mux when psock is being unattached. */ 191 192#define SAVE_PSOCK_STATS(_stat) (agg_stats->_stat += stats->_stat) 193 SAVE_PSOCK_STATS(rx_msgs); 194 SAVE_PSOCK_STATS(rx_bytes); 195 SAVE_PSOCK_STATS(rx_aborts); 196 SAVE_PSOCK_STATS(rx_mem_fail); 197 SAVE_PSOCK_STATS(rx_need_more_hdr); 198 SAVE_PSOCK_STATS(rx_msg_too_big); 199 SAVE_PSOCK_STATS(rx_msg_timeouts); 200 SAVE_PSOCK_STATS(rx_bad_hdr_len); 201 SAVE_PSOCK_STATS(tx_msgs); 202 SAVE_PSOCK_STATS(tx_bytes); 203 SAVE_PSOCK_STATS(reserved); 204 SAVE_PSOCK_STATS(unreserved); 205 SAVE_PSOCK_STATS(tx_aborts); 206#undef SAVE_PSOCK_STATS 207} 208 209static inline void aggregate_mux_stats(struct kcm_mux_stats *stats, 210 struct kcm_mux_stats *agg_stats) 211{ 212 /* Save psock statistics in the mux when psock is being unattached. */ 213 214#define SAVE_MUX_STATS(_stat) (agg_stats->_stat += stats->_stat) 215 SAVE_MUX_STATS(rx_msgs); 216 SAVE_MUX_STATS(rx_bytes); 217 SAVE_MUX_STATS(tx_msgs); 218 SAVE_MUX_STATS(tx_bytes); 219 SAVE_MUX_STATS(rx_ready_drops); 220 SAVE_MUX_STATS(psock_attach); 221 SAVE_MUX_STATS(psock_unattach_rsvd); 222 SAVE_MUX_STATS(psock_unattach); 223#undef SAVE_MUX_STATS 224} 225 226#endif /* __NET_KCM_H_ */ 227