1
2
3
4
5
6
7
8
9
10
11#include <linux/kernel.h>
12#include <linux/kmemcheck.h>
13#include <linux/slab.h>
14#include <linux/module.h>
15#include <net/inet_hashtables.h>
16#include <net/inet_timewait_sock.h>
17#include <net/ip.h>
18
19
20
21
22
23
24
25
26
27
28int inet_twsk_unhash(struct inet_timewait_sock *tw)
29{
30 if (hlist_nulls_unhashed(&tw->tw_node))
31 return 0;
32
33 hlist_nulls_del_rcu(&tw->tw_node);
34 sk_nulls_node_init(&tw->tw_node);
35
36
37
38
39 return 1;
40}
41
42
43
44
45
46
47
48
49
50
51int inet_twsk_bind_unhash(struct inet_timewait_sock *tw,
52 struct inet_hashinfo *hashinfo)
53{
54 struct inet_bind_bucket *tb = tw->tw_tb;
55
56 if (!tb)
57 return 0;
58
59 __hlist_del(&tw->tw_bind_node);
60 tw->tw_tb = NULL;
61 inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb);
62
63
64
65
66 return 1;
67}
68
69
70static void inet_twsk_kill(struct inet_timewait_sock *tw)
71{
72 struct inet_hashinfo *hashinfo = tw->tw_dr->hashinfo;
73 struct inet_bind_hashbucket *bhead;
74 int refcnt;
75
76 spinlock_t *lock = inet_ehash_lockp(hashinfo, tw->tw_hash);
77
78 spin_lock(lock);
79 refcnt = inet_twsk_unhash(tw);
80 spin_unlock(lock);
81
82
83 bhead = &hashinfo->bhash[inet_bhashfn(twsk_net(tw), tw->tw_num,
84 hashinfo->bhash_size)];
85
86 spin_lock(&bhead->lock);
87 refcnt += inet_twsk_bind_unhash(tw, hashinfo);
88 spin_unlock(&bhead->lock);
89
90 BUG_ON(refcnt >= atomic_read(&tw->tw_refcnt));
91 atomic_sub(refcnt, &tw->tw_refcnt);
92 atomic_dec(&tw->tw_dr->tw_count);
93 inet_twsk_put(tw);
94}
95
96void inet_twsk_free(struct inet_timewait_sock *tw)
97{
98 struct module *owner = tw->tw_prot->owner;
99 twsk_destructor((struct sock *)tw);
100#ifdef SOCK_REFCNT_DEBUG
101 pr_debug("%s timewait_sock %p released\n", tw->tw_prot->name, tw);
102#endif
103 kmem_cache_free(tw->tw_prot->twsk_prot->twsk_slab, tw);
104 module_put(owner);
105}
106
107void inet_twsk_put(struct inet_timewait_sock *tw)
108{
109 if (atomic_dec_and_test(&tw->tw_refcnt))
110 inet_twsk_free(tw);
111}
112EXPORT_SYMBOL_GPL(inet_twsk_put);
113
114static void inet_twsk_add_node_rcu(struct inet_timewait_sock *tw,
115 struct hlist_nulls_head *list)
116{
117 hlist_nulls_add_head_rcu(&tw->tw_node, list);
118}
119
120static void inet_twsk_add_bind_node(struct inet_timewait_sock *tw,
121 struct hlist_head *list)
122{
123 hlist_add_head(&tw->tw_bind_node, list);
124}
125
126
127
128
129
130
131void __inet_twsk_hashdance(struct inet_timewait_sock *tw, struct sock *sk,
132 struct inet_hashinfo *hashinfo)
133{
134 const struct inet_sock *inet = inet_sk(sk);
135 const struct inet_connection_sock *icsk = inet_csk(sk);
136 struct inet_ehash_bucket *ehead = inet_ehash_bucket(hashinfo, sk->sk_hash);
137 spinlock_t *lock = inet_ehash_lockp(hashinfo, sk->sk_hash);
138 struct inet_bind_hashbucket *bhead;
139
140
141
142
143 bhead = &hashinfo->bhash[inet_bhashfn(twsk_net(tw), inet->inet_num,
144 hashinfo->bhash_size)];
145 spin_lock(&bhead->lock);
146 tw->tw_tb = icsk->icsk_bind_hash;
147 WARN_ON(!icsk->icsk_bind_hash);
148 inet_twsk_add_bind_node(tw, &tw->tw_tb->owners);
149 spin_unlock(&bhead->lock);
150
151 spin_lock(lock);
152
153
154
155
156
157
158
159
160
161
162 atomic_set(&tw->tw_refcnt, 1 + 1 + 1);
163 inet_twsk_add_node_rcu(tw, &ehead->chain);
164
165
166 if (__sk_nulls_del_node_init_rcu(sk))
167 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
168
169 spin_unlock(lock);
170}
171EXPORT_SYMBOL_GPL(__inet_twsk_hashdance);
172
173static void tw_timer_handler(unsigned long data)
174{
175 struct inet_timewait_sock *tw = (struct inet_timewait_sock *)data;
176
177 if (tw->tw_kill)
178 NET_INC_STATS_BH(twsk_net(tw), LINUX_MIB_TIMEWAITKILLED);
179 else
180 NET_INC_STATS_BH(twsk_net(tw), LINUX_MIB_TIMEWAITED);
181 inet_twsk_kill(tw);
182}
183
184struct inet_timewait_sock *inet_twsk_alloc(const struct sock *sk,
185 struct inet_timewait_death_row *dr,
186 const int state)
187{
188 struct inet_timewait_sock *tw;
189
190 if (atomic_read(&dr->tw_count) >= dr->sysctl_max_tw_buckets)
191 return NULL;
192
193 tw = kmem_cache_alloc(sk->sk_prot_creator->twsk_prot->twsk_slab,
194 GFP_ATOMIC);
195 if (tw) {
196 const struct inet_sock *inet = inet_sk(sk);
197
198 kmemcheck_annotate_bitfield(tw, flags);
199
200 tw->tw_dr = dr;
201
202 tw->tw_daddr = inet->inet_daddr;
203 tw->tw_rcv_saddr = inet->inet_rcv_saddr;
204 tw->tw_bound_dev_if = sk->sk_bound_dev_if;
205 tw->tw_tos = inet->tos;
206 tw->tw_num = inet->inet_num;
207 tw->tw_state = TCP_TIME_WAIT;
208 tw->tw_substate = state;
209 tw->tw_sport = inet->inet_sport;
210 tw->tw_dport = inet->inet_dport;
211 tw->tw_family = sk->sk_family;
212 tw->tw_reuse = sk->sk_reuse;
213 tw->tw_hash = sk->sk_hash;
214 tw->tw_ipv6only = 0;
215 tw->tw_transparent = inet->transparent;
216 tw->tw_prot = sk->sk_prot_creator;
217 atomic64_set(&tw->tw_cookie, atomic64_read(&sk->sk_cookie));
218 twsk_net_set(tw, sock_net(sk));
219 setup_timer(&tw->tw_timer, tw_timer_handler, (unsigned long)tw);
220
221
222
223
224
225 atomic_set(&tw->tw_refcnt, 0);
226
227 __module_get(tw->tw_prot->owner);
228 }
229
230 return tw;
231}
232EXPORT_SYMBOL_GPL(inet_twsk_alloc);
233
234
235
236
237
238
239void inet_twsk_deschedule(struct inet_timewait_sock *tw)
240{
241 if (del_timer_sync(&tw->tw_timer))
242 inet_twsk_kill(tw);
243}
244EXPORT_SYMBOL(inet_twsk_deschedule);
245
246void inet_twsk_schedule(struct inet_timewait_sock *tw, const int timeo)
247{
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273 tw->tw_kill = timeo <= 4*HZ;
274 if (!mod_timer_pinned(&tw->tw_timer, jiffies + timeo)) {
275 atomic_inc(&tw->tw_refcnt);
276 atomic_inc(&tw->tw_dr->tw_count);
277 }
278}
279EXPORT_SYMBOL_GPL(inet_twsk_schedule);
280
281void inet_twsk_purge(struct inet_hashinfo *hashinfo,
282 struct inet_timewait_death_row *twdr, int family)
283{
284 struct inet_timewait_sock *tw;
285 struct sock *sk;
286 struct hlist_nulls_node *node;
287 unsigned int slot;
288
289 for (slot = 0; slot <= hashinfo->ehash_mask; slot++) {
290 struct inet_ehash_bucket *head = &hashinfo->ehash[slot];
291restart_rcu:
292 cond_resched();
293 rcu_read_lock();
294restart:
295 sk_nulls_for_each_rcu(sk, node, &head->chain) {
296 if (sk->sk_state != TCP_TIME_WAIT)
297 continue;
298 tw = inet_twsk(sk);
299 if ((tw->tw_family != family) ||
300 atomic_read(&twsk_net(tw)->count))
301 continue;
302
303 if (unlikely(!atomic_inc_not_zero(&tw->tw_refcnt)))
304 continue;
305
306 if (unlikely((tw->tw_family != family) ||
307 atomic_read(&twsk_net(tw)->count))) {
308 inet_twsk_put(tw);
309 goto restart;
310 }
311
312 rcu_read_unlock();
313 local_bh_disable();
314 inet_twsk_deschedule(tw);
315 local_bh_enable();
316 inet_twsk_put(tw);
317 goto restart_rcu;
318 }
319
320
321
322
323 if (get_nulls_value(node) != slot)
324 goto restart;
325 rcu_read_unlock();
326 }
327}
328EXPORT_SYMBOL_GPL(inet_twsk_purge);
329