1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71#include <linux/kernel.h>
72#include <linux/string.h>
73#include <linux/socket.h>
74#include <linux/un.h>
75#include <linux/net.h>
76#include <linux/fs.h>
77#include <linux/skbuff.h>
78#include <linux/netdevice.h>
79#include <linux/file.h>
80#include <linux/proc_fs.h>
81#include <linux/mutex.h>
82#include <linux/wait.h>
83
84#include <net/sock.h>
85#include <net/af_unix.h>
86#include <net/scm.h>
87#include <net/tcp_states.h>
88
89
90
91static LIST_HEAD(gc_inflight_list);
92static LIST_HEAD(gc_candidates);
93static DEFINE_SPINLOCK(unix_gc_lock);
94static DECLARE_WAIT_QUEUE_HEAD(unix_gc_wait);
95
96unsigned int unix_tot_inflight;
97
98struct sock *unix_get_socket(struct file *filp)
99{
100 struct sock *u_sock = NULL;
101 struct inode *inode = file_inode(filp);
102
103
104 if (S_ISSOCK(inode->i_mode) && !(filp->f_mode & FMODE_PATH)) {
105 struct socket *sock = SOCKET_I(inode);
106 struct sock *s = sock->sk;
107
108
109 if (s && sock->ops && sock->ops->family == PF_UNIX)
110 u_sock = s;
111 }
112 return u_sock;
113}
114
115
116
117
118
119void unix_inflight(struct file *fp)
120{
121 struct sock *s = unix_get_socket(fp);
122
123 if (s) {
124 struct unix_sock *u = unix_sk(s);
125
126 spin_lock(&unix_gc_lock);
127
128 if (atomic_long_inc_return(&u->inflight) == 1) {
129 BUG_ON(!list_empty(&u->link));
130 list_add_tail(&u->link, &gc_inflight_list);
131 } else {
132 BUG_ON(list_empty(&u->link));
133 }
134 unix_tot_inflight++;
135 spin_unlock(&unix_gc_lock);
136 }
137}
138
139void unix_notinflight(struct file *fp)
140{
141 struct sock *s = unix_get_socket(fp);
142
143 if (s) {
144 struct unix_sock *u = unix_sk(s);
145
146 spin_lock(&unix_gc_lock);
147 BUG_ON(list_empty(&u->link));
148
149 if (atomic_long_dec_and_test(&u->inflight))
150 list_del_init(&u->link);
151 unix_tot_inflight--;
152 spin_unlock(&unix_gc_lock);
153 }
154}
155
156static void scan_inflight(struct sock *x, void (*func)(struct unix_sock *),
157 struct sk_buff_head *hitlist)
158{
159 struct sk_buff *skb;
160 struct sk_buff *next;
161
162 spin_lock(&x->sk_receive_queue.lock);
163 skb_queue_walk_safe(&x->sk_receive_queue, skb, next) {
164
165 if (UNIXCB(skb).fp) {
166 bool hit = false;
167
168 int nfd = UNIXCB(skb).fp->count;
169 struct file **fp = UNIXCB(skb).fp->fp;
170
171 while (nfd--) {
172
173 struct sock *sk = unix_get_socket(*fp++);
174
175 if (sk) {
176 struct unix_sock *u = unix_sk(sk);
177
178
179
180
181
182 if (test_bit(UNIX_GC_CANDIDATE, &u->gc_flags)) {
183 hit = true;
184
185 func(u);
186 }
187 }
188 }
189 if (hit && hitlist != NULL) {
190 __skb_unlink(skb, &x->sk_receive_queue);
191 __skb_queue_tail(hitlist, skb);
192 }
193 }
194 }
195 spin_unlock(&x->sk_receive_queue.lock);
196}
197
198static void scan_children(struct sock *x, void (*func)(struct unix_sock *),
199 struct sk_buff_head *hitlist)
200{
201 if (x->sk_state != TCP_LISTEN) {
202 scan_inflight(x, func, hitlist);
203 } else {
204 struct sk_buff *skb;
205 struct sk_buff *next;
206 struct unix_sock *u;
207 LIST_HEAD(embryos);
208
209
210
211
212 spin_lock(&x->sk_receive_queue.lock);
213 skb_queue_walk_safe(&x->sk_receive_queue, skb, next) {
214 u = unix_sk(skb->sk);
215
216
217
218
219 BUG_ON(!list_empty(&u->link));
220 list_add_tail(&u->link, &embryos);
221 }
222 spin_unlock(&x->sk_receive_queue.lock);
223
224 while (!list_empty(&embryos)) {
225 u = list_entry(embryos.next, struct unix_sock, link);
226 scan_inflight(&u->sk, func, hitlist);
227 list_del_init(&u->link);
228 }
229 }
230}
231
232static void dec_inflight(struct unix_sock *usk)
233{
234 atomic_long_dec(&usk->inflight);
235}
236
237static void inc_inflight(struct unix_sock *usk)
238{
239 atomic_long_inc(&usk->inflight);
240}
241
242static void inc_inflight_move_tail(struct unix_sock *u)
243{
244 atomic_long_inc(&u->inflight);
245
246
247
248
249 if (test_bit(UNIX_GC_MAYBE_CYCLE, &u->gc_flags))
250 list_move_tail(&u->link, &gc_candidates);
251}
252
253static bool gc_in_progress;
254#define UNIX_INFLIGHT_TRIGGER_GC 16000
255
256void wait_for_unix_gc(void)
257{
258
259
260
261 if (unix_tot_inflight > UNIX_INFLIGHT_TRIGGER_GC && !gc_in_progress)
262 unix_gc();
263 wait_event(unix_gc_wait, gc_in_progress == false);
264}
265
266
267void unix_gc(void)
268{
269 struct unix_sock *u;
270 struct unix_sock *next;
271 struct sk_buff_head hitlist;
272 struct list_head cursor;
273 LIST_HEAD(not_cycle_list);
274
275 spin_lock(&unix_gc_lock);
276
277
278 if (gc_in_progress)
279 goto out;
280
281 gc_in_progress = true;
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297 list_for_each_entry_safe(u, next, &gc_inflight_list, link) {
298 long total_refs;
299 long inflight_refs;
300
301 total_refs = file_count(u->sk.sk_socket->file);
302 inflight_refs = atomic_long_read(&u->inflight);
303
304 BUG_ON(inflight_refs < 1);
305 BUG_ON(total_refs < inflight_refs);
306 if (total_refs == inflight_refs) {
307 list_move_tail(&u->link, &gc_candidates);
308 __set_bit(UNIX_GC_CANDIDATE, &u->gc_flags);
309 __set_bit(UNIX_GC_MAYBE_CYCLE, &u->gc_flags);
310 }
311 }
312
313
314
315
316 list_for_each_entry(u, &gc_candidates, link)
317 scan_children(&u->sk, dec_inflight, NULL);
318
319
320
321
322
323
324
325
326 list_add(&cursor, &gc_candidates);
327 while (cursor.next != &gc_candidates) {
328 u = list_entry(cursor.next, struct unix_sock, link);
329
330
331 list_move(&cursor, &u->link);
332
333 if (atomic_long_read(&u->inflight) > 0) {
334 list_move_tail(&u->link, ¬_cycle_list);
335 __clear_bit(UNIX_GC_MAYBE_CYCLE, &u->gc_flags);
336 scan_children(&u->sk, inc_inflight_move_tail, NULL);
337 }
338 }
339 list_del(&cursor);
340
341
342
343
344 while (!list_empty(¬_cycle_list)) {
345 u = list_entry(not_cycle_list.next, struct unix_sock, link);
346 __clear_bit(UNIX_GC_CANDIDATE, &u->gc_flags);
347 list_move_tail(&u->link, &gc_inflight_list);
348 }
349
350
351
352
353
354 skb_queue_head_init(&hitlist);
355 list_for_each_entry(u, &gc_candidates, link)
356 scan_children(&u->sk, inc_inflight, &hitlist);
357
358 spin_unlock(&unix_gc_lock);
359
360
361 __skb_queue_purge(&hitlist);
362
363 spin_lock(&unix_gc_lock);
364
365
366 BUG_ON(!list_empty(&gc_candidates));
367 gc_in_progress = false;
368 wake_up(&unix_gc_wait);
369
370 out:
371 spin_unlock(&unix_gc_lock);
372}
373