1
2
3
4
5
6
7
8
9
10
11#include <linux/kernel.h>
12#include <linux/kmemcheck.h>
13#include <linux/slab.h>
14#include <linux/module.h>
15#include <net/inet_hashtables.h>
16#include <net/inet_timewait_sock.h>
17#include <net/ip.h>
18
19
20
21
22
23
24
25
26
27
28
29void inet_twsk_bind_unhash(struct inet_timewait_sock *tw,
30 struct inet_hashinfo *hashinfo)
31{
32 struct inet_bind_bucket *tb = tw->tw_tb;
33
34 if (!tb)
35 return;
36
37 __hlist_del(&tw->tw_bind_node);
38 tw->tw_tb = NULL;
39 inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb);
40 __sock_put((struct sock *)tw);
41}
42
43
44static void inet_twsk_kill(struct inet_timewait_sock *tw)
45{
46 struct inet_hashinfo *hashinfo = tw->tw_dr->hashinfo;
47 spinlock_t *lock = inet_ehash_lockp(hashinfo, tw->tw_hash);
48 struct inet_bind_hashbucket *bhead;
49
50 spin_lock(lock);
51 sk_nulls_del_node_init_rcu((struct sock *)tw);
52 spin_unlock(lock);
53
54
55 bhead = &hashinfo->bhash[inet_bhashfn(twsk_net(tw), tw->tw_num,
56 hashinfo->bhash_size)];
57
58 spin_lock(&bhead->lock);
59 inet_twsk_bind_unhash(tw, hashinfo);
60 spin_unlock(&bhead->lock);
61
62 atomic_dec(&tw->tw_dr->tw_count);
63 inet_twsk_put(tw);
64}
65
66void inet_twsk_free(struct inet_timewait_sock *tw)
67{
68 struct module *owner = tw->tw_prot->owner;
69 twsk_destructor((struct sock *)tw);
70#ifdef SOCK_REFCNT_DEBUG
71 pr_debug("%s timewait_sock %p released\n", tw->tw_prot->name, tw);
72#endif
73 kmem_cache_free(tw->tw_prot->twsk_prot->twsk_slab, tw);
74 module_put(owner);
75}
76
77void inet_twsk_put(struct inet_timewait_sock *tw)
78{
79 if (atomic_dec_and_test(&tw->tw_refcnt))
80 inet_twsk_free(tw);
81}
82EXPORT_SYMBOL_GPL(inet_twsk_put);
83
84static void inet_twsk_add_node_rcu(struct inet_timewait_sock *tw,
85 struct hlist_nulls_head *list)
86{
87 hlist_nulls_add_head_rcu(&tw->tw_node, list);
88}
89
90static void inet_twsk_add_bind_node(struct inet_timewait_sock *tw,
91 struct hlist_head *list)
92{
93 hlist_add_head(&tw->tw_bind_node, list);
94}
95
96
97
98
99
100
101void __inet_twsk_hashdance(struct inet_timewait_sock *tw, struct sock *sk,
102 struct inet_hashinfo *hashinfo)
103{
104 const struct inet_sock *inet = inet_sk(sk);
105 const struct inet_connection_sock *icsk = inet_csk(sk);
106 struct inet_ehash_bucket *ehead = inet_ehash_bucket(hashinfo, sk->sk_hash);
107 spinlock_t *lock = inet_ehash_lockp(hashinfo, sk->sk_hash);
108 struct inet_bind_hashbucket *bhead;
109
110
111
112
113 bhead = &hashinfo->bhash[inet_bhashfn(twsk_net(tw), inet->inet_num,
114 hashinfo->bhash_size)];
115 spin_lock(&bhead->lock);
116 tw->tw_tb = icsk->icsk_bind_hash;
117 WARN_ON(!icsk->icsk_bind_hash);
118 inet_twsk_add_bind_node(tw, &tw->tw_tb->owners);
119 spin_unlock(&bhead->lock);
120
121 spin_lock(lock);
122
123
124
125
126
127
128
129
130
131
132
133
134 atomic_set(&tw->tw_refcnt, 4);
135 inet_twsk_add_node_rcu(tw, &ehead->chain);
136
137
138 if (__sk_nulls_del_node_init_rcu(sk))
139 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
140
141 spin_unlock(lock);
142}
143EXPORT_SYMBOL_GPL(__inet_twsk_hashdance);
144
145static void tw_timer_handler(unsigned long data)
146{
147 struct inet_timewait_sock *tw = (struct inet_timewait_sock *)data;
148
149 if (tw->tw_kill)
150 NET_INC_STATS_BH(twsk_net(tw), LINUX_MIB_TIMEWAITKILLED);
151 else
152 NET_INC_STATS_BH(twsk_net(tw), LINUX_MIB_TIMEWAITED);
153 inet_twsk_kill(tw);
154}
155
156struct inet_timewait_sock *inet_twsk_alloc(const struct sock *sk,
157 struct inet_timewait_death_row *dr,
158 const int state)
159{
160 struct inet_timewait_sock *tw;
161
162 if (atomic_read(&dr->tw_count) >= dr->sysctl_max_tw_buckets)
163 return NULL;
164
165 tw = kmem_cache_alloc(sk->sk_prot_creator->twsk_prot->twsk_slab,
166 GFP_ATOMIC);
167 if (tw) {
168 const struct inet_sock *inet = inet_sk(sk);
169
170 kmemcheck_annotate_bitfield(tw, flags);
171
172 tw->tw_dr = dr;
173
174 tw->tw_daddr = inet->inet_daddr;
175 tw->tw_rcv_saddr = inet->inet_rcv_saddr;
176 tw->tw_bound_dev_if = sk->sk_bound_dev_if;
177 tw->tw_tos = inet->tos;
178 tw->tw_num = inet->inet_num;
179 tw->tw_state = TCP_TIME_WAIT;
180 tw->tw_substate = state;
181 tw->tw_sport = inet->inet_sport;
182 tw->tw_dport = inet->inet_dport;
183 tw->tw_family = sk->sk_family;
184 tw->tw_reuse = sk->sk_reuse;
185 tw->tw_hash = sk->sk_hash;
186 tw->tw_ipv6only = 0;
187 tw->tw_transparent = inet->transparent;
188 tw->tw_prot = sk->sk_prot_creator;
189 atomic64_set(&tw->tw_cookie, atomic64_read(&sk->sk_cookie));
190 twsk_net_set(tw, sock_net(sk));
191 setup_timer(&tw->tw_timer, tw_timer_handler, (unsigned long)tw);
192
193
194
195
196
197 atomic_set(&tw->tw_refcnt, 0);
198
199 __module_get(tw->tw_prot->owner);
200 }
201
202 return tw;
203}
204EXPORT_SYMBOL_GPL(inet_twsk_alloc);
205
206
207
208
209
210
211
212
213
214void inet_twsk_deschedule_put(struct inet_timewait_sock *tw)
215{
216 if (del_timer_sync(&tw->tw_timer))
217 inet_twsk_kill(tw);
218 inet_twsk_put(tw);
219}
220EXPORT_SYMBOL(inet_twsk_deschedule_put);
221
222void __inet_twsk_schedule(struct inet_timewait_sock *tw, int timeo, bool rearm)
223{
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249 tw->tw_kill = timeo <= 4*HZ;
250 if (!rearm) {
251 BUG_ON(mod_timer_pinned(&tw->tw_timer, jiffies + timeo));
252 atomic_inc(&tw->tw_dr->tw_count);
253 } else {
254 mod_timer_pending(&tw->tw_timer, jiffies + timeo);
255 }
256}
257EXPORT_SYMBOL_GPL(__inet_twsk_schedule);
258
259void inet_twsk_purge(struct inet_hashinfo *hashinfo,
260 struct inet_timewait_death_row *twdr, int family)
261{
262 struct inet_timewait_sock *tw;
263 struct sock *sk;
264 struct hlist_nulls_node *node;
265 unsigned int slot;
266
267 for (slot = 0; slot <= hashinfo->ehash_mask; slot++) {
268 struct inet_ehash_bucket *head = &hashinfo->ehash[slot];
269restart_rcu:
270 cond_resched();
271 rcu_read_lock();
272restart:
273 sk_nulls_for_each_rcu(sk, node, &head->chain) {
274 if (sk->sk_state != TCP_TIME_WAIT)
275 continue;
276 tw = inet_twsk(sk);
277 if ((tw->tw_family != family) ||
278 atomic_read(&twsk_net(tw)->count))
279 continue;
280
281 if (unlikely(!atomic_inc_not_zero(&tw->tw_refcnt)))
282 continue;
283
284 if (unlikely((tw->tw_family != family) ||
285 atomic_read(&twsk_net(tw)->count))) {
286 inet_twsk_put(tw);
287 goto restart;
288 }
289
290 rcu_read_unlock();
291 local_bh_disable();
292 inet_twsk_deschedule_put(tw);
293 local_bh_enable();
294 goto restart_rcu;
295 }
296
297
298
299
300 if (get_nulls_value(node) != slot)
301 goto restart;
302 rcu_read_unlock();
303 }
304}
305EXPORT_SYMBOL_GPL(inet_twsk_purge);
306