1
2
3
4
5
6
7
8
9
10
11
12#include <linux/kernel.h>
13#include <linux/slab.h>
14#include <linux/module.h>
15#include <net/inet_hashtables.h>
16#include <net/inet_timewait_sock.h>
17#include <net/ip.h>
18
19
20
21
22
23
24
25
26
27
28
29void inet_twsk_bind_unhash(struct inet_timewait_sock *tw,
30 struct inet_hashinfo *hashinfo)
31{
32 struct inet_bind_bucket *tb = tw->tw_tb;
33
34 if (!tb)
35 return;
36
37 __hlist_del(&tw->tw_bind_node);
38 tw->tw_tb = NULL;
39 inet_bind_bucket_destroy(hashinfo->bind_bucket_cachep, tb);
40 __sock_put((struct sock *)tw);
41}
42
43
44static void inet_twsk_kill(struct inet_timewait_sock *tw)
45{
46 struct inet_hashinfo *hashinfo = tw->tw_dr->hashinfo;
47 spinlock_t *lock = inet_ehash_lockp(hashinfo, tw->tw_hash);
48 struct inet_bind_hashbucket *bhead;
49
50 spin_lock(lock);
51 sk_nulls_del_node_init_rcu((struct sock *)tw);
52 spin_unlock(lock);
53
54
55 bhead = &hashinfo->bhash[inet_bhashfn(twsk_net(tw), tw->tw_num,
56 hashinfo->bhash_size)];
57
58 spin_lock(&bhead->lock);
59 inet_twsk_bind_unhash(tw, hashinfo);
60 spin_unlock(&bhead->lock);
61
62 if (refcount_dec_and_test(&tw->tw_dr->tw_refcount))
63 kfree(tw->tw_dr);
64
65 inet_twsk_put(tw);
66}
67
68void inet_twsk_free(struct inet_timewait_sock *tw)
69{
70 struct module *owner = tw->tw_prot->owner;
71 twsk_destructor((struct sock *)tw);
72#ifdef SOCK_REFCNT_DEBUG
73 pr_debug("%s timewait_sock %p released\n", tw->tw_prot->name, tw);
74#endif
75 kmem_cache_free(tw->tw_prot->twsk_prot->twsk_slab, tw);
76 module_put(owner);
77}
78
79void inet_twsk_put(struct inet_timewait_sock *tw)
80{
81 if (refcount_dec_and_test(&tw->tw_refcnt))
82 inet_twsk_free(tw);
83}
84EXPORT_SYMBOL_GPL(inet_twsk_put);
85
86static void inet_twsk_add_node_rcu(struct inet_timewait_sock *tw,
87 struct hlist_nulls_head *list)
88{
89 hlist_nulls_add_head_rcu(&tw->tw_node, list);
90}
91
92static void inet_twsk_add_bind_node(struct inet_timewait_sock *tw,
93 struct hlist_head *list)
94{
95 hlist_add_head(&tw->tw_bind_node, list);
96}
97
98
99
100
101
102
103void inet_twsk_hashdance(struct inet_timewait_sock *tw, struct sock *sk,
104 struct inet_hashinfo *hashinfo)
105{
106 const struct inet_sock *inet = inet_sk(sk);
107 const struct inet_connection_sock *icsk = inet_csk(sk);
108 struct inet_ehash_bucket *ehead = inet_ehash_bucket(hashinfo, sk->sk_hash);
109 spinlock_t *lock = inet_ehash_lockp(hashinfo, sk->sk_hash);
110 struct inet_bind_hashbucket *bhead;
111
112
113
114
115 bhead = &hashinfo->bhash[inet_bhashfn(twsk_net(tw), inet->inet_num,
116 hashinfo->bhash_size)];
117 spin_lock(&bhead->lock);
118 tw->tw_tb = icsk->icsk_bind_hash;
119 WARN_ON(!icsk->icsk_bind_hash);
120 inet_twsk_add_bind_node(tw, &tw->tw_tb->owners);
121 spin_unlock(&bhead->lock);
122
123 spin_lock(lock);
124
125 inet_twsk_add_node_rcu(tw, &ehead->chain);
126
127
128 if (__sk_nulls_del_node_init_rcu(sk))
129 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
130
131 spin_unlock(lock);
132
133
134
135
136
137
138
139
140
141
142 refcount_set(&tw->tw_refcnt, 3);
143}
144EXPORT_SYMBOL_GPL(inet_twsk_hashdance);
145
146static void tw_timer_handler(struct timer_list *t)
147{
148 struct inet_timewait_sock *tw = from_timer(tw, t, tw_timer);
149
150 inet_twsk_kill(tw);
151}
152
153struct inet_timewait_sock *inet_twsk_alloc(const struct sock *sk,
154 struct inet_timewait_death_row *dr,
155 const int state)
156{
157 struct inet_timewait_sock *tw;
158
159 if (refcount_read(&dr->tw_refcount) - 1 >=
160 READ_ONCE(dr->sysctl_max_tw_buckets))
161 return NULL;
162
163 tw = kmem_cache_alloc(sk->sk_prot_creator->twsk_prot->twsk_slab,
164 GFP_ATOMIC);
165 if (tw) {
166 const struct inet_sock *inet = inet_sk(sk);
167
168 tw->tw_dr = dr;
169
170 tw->tw_daddr = inet->inet_daddr;
171 tw->tw_rcv_saddr = inet->inet_rcv_saddr;
172 tw->tw_bound_dev_if = sk->sk_bound_dev_if;
173 tw->tw_tos = inet->tos;
174 tw->tw_num = inet->inet_num;
175 tw->tw_state = TCP_TIME_WAIT;
176 tw->tw_substate = state;
177 tw->tw_sport = inet->inet_sport;
178 tw->tw_dport = inet->inet_dport;
179 tw->tw_family = sk->sk_family;
180 tw->tw_reuse = sk->sk_reuse;
181 tw->tw_reuseport = sk->sk_reuseport;
182 tw->tw_hash = sk->sk_hash;
183 tw->tw_ipv6only = 0;
184 tw->tw_transparent = inet->transparent;
185 tw->tw_prot = sk->sk_prot_creator;
186 atomic64_set(&tw->tw_cookie, atomic64_read(&sk->sk_cookie));
187 twsk_net_set(tw, sock_net(sk));
188 timer_setup(&tw->tw_timer, tw_timer_handler, TIMER_PINNED);
189
190
191
192
193
194 refcount_set(&tw->tw_refcnt, 0);
195
196 __module_get(tw->tw_prot->owner);
197 }
198
199 return tw;
200}
201EXPORT_SYMBOL_GPL(inet_twsk_alloc);
202
203
204
205
206
207
208
209
210
211void inet_twsk_deschedule_put(struct inet_timewait_sock *tw)
212{
213 if (del_timer_sync(&tw->tw_timer))
214 inet_twsk_kill(tw);
215 inet_twsk_put(tw);
216}
217EXPORT_SYMBOL(inet_twsk_deschedule_put);
218
219void __inet_twsk_schedule(struct inet_timewait_sock *tw, int timeo, bool rearm)
220{
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246 if (!rearm) {
247 bool kill = timeo <= 4*HZ;
248
249 __NET_INC_STATS(twsk_net(tw), kill ? LINUX_MIB_TIMEWAITKILLED :
250 LINUX_MIB_TIMEWAITED);
251 BUG_ON(mod_timer(&tw->tw_timer, jiffies + timeo));
252 refcount_inc(&tw->tw_dr->tw_refcount);
253 } else {
254 mod_timer_pending(&tw->tw_timer, jiffies + timeo);
255 }
256}
257EXPORT_SYMBOL_GPL(__inet_twsk_schedule);
258
259void inet_twsk_purge(struct inet_hashinfo *hashinfo, int family)
260{
261 struct inet_timewait_sock *tw;
262 struct sock *sk;
263 struct hlist_nulls_node *node;
264 unsigned int slot;
265
266 for (slot = 0; slot <= hashinfo->ehash_mask; slot++) {
267 struct inet_ehash_bucket *head = &hashinfo->ehash[slot];
268restart_rcu:
269 cond_resched();
270 rcu_read_lock();
271restart:
272 sk_nulls_for_each_rcu(sk, node, &head->chain) {
273 if (sk->sk_state != TCP_TIME_WAIT)
274 continue;
275 tw = inet_twsk(sk);
276 if ((tw->tw_family != family) ||
277 refcount_read(&twsk_net(tw)->ns.count))
278 continue;
279
280 if (unlikely(!refcount_inc_not_zero(&tw->tw_refcnt)))
281 continue;
282
283 if (unlikely((tw->tw_family != family) ||
284 refcount_read(&twsk_net(tw)->ns.count))) {
285 inet_twsk_put(tw);
286 goto restart;
287 }
288
289 rcu_read_unlock();
290 local_bh_disable();
291 inet_twsk_deschedule_put(tw);
292 local_bh_enable();
293 goto restart_rcu;
294 }
295
296
297
298
299 if (get_nulls_value(node) != slot)
300 goto restart;
301 rcu_read_unlock();
302 }
303}
304EXPORT_SYMBOL_GPL(inet_twsk_purge);
305