1
2
3
4
5
6
7
8
9
10
11
12
13
14#include <linux/list.h>
15#include <linux/spinlock.h>
16#include <linux/module.h>
17#include <linux/timer.h>
18#include <linux/mm.h>
19#include <linux/random.h>
20#include <linux/skbuff.h>
21#include <linux/rtnetlink.h>
22#include <linux/slab.h>
23
24#include <net/sock.h>
25#include <net/inet_frag.h>
26#include <net/inet_ecn.h>
27
28
29
30
31
32const u8 ip_frag_ecn_table[16] = {
33
34 [IPFRAG_ECN_CE | IPFRAG_ECN_ECT_0] = INET_ECN_CE,
35 [IPFRAG_ECN_CE | IPFRAG_ECN_ECT_1] = INET_ECN_CE,
36 [IPFRAG_ECN_CE | IPFRAG_ECN_ECT_0 | IPFRAG_ECN_ECT_1] = INET_ECN_CE,
37
38
39 [IPFRAG_ECN_NOT_ECT | IPFRAG_ECN_CE] = 0xff,
40 [IPFRAG_ECN_NOT_ECT | IPFRAG_ECN_ECT_0] = 0xff,
41 [IPFRAG_ECN_NOT_ECT | IPFRAG_ECN_ECT_1] = 0xff,
42 [IPFRAG_ECN_NOT_ECT | IPFRAG_ECN_ECT_0 | IPFRAG_ECN_ECT_1] = 0xff,
43 [IPFRAG_ECN_NOT_ECT | IPFRAG_ECN_CE | IPFRAG_ECN_ECT_0] = 0xff,
44 [IPFRAG_ECN_NOT_ECT | IPFRAG_ECN_CE | IPFRAG_ECN_ECT_1] = 0xff,
45 [IPFRAG_ECN_NOT_ECT | IPFRAG_ECN_CE | IPFRAG_ECN_ECT_0 | IPFRAG_ECN_ECT_1] = 0xff,
46};
47EXPORT_SYMBOL(ip_frag_ecn_table);
48
49static void inet_frag_secret_rebuild(unsigned long dummy)
50{
51 struct inet_frags *f = (struct inet_frags *)dummy;
52 unsigned long now = jiffies;
53 int i;
54
55
56 write_lock(&f->lock);
57
58 get_random_bytes(&f->rnd, sizeof(u32));
59 for (i = 0; i < INETFRAGS_HASHSZ; i++) {
60 struct inet_frag_bucket *hb;
61 struct inet_frag_queue *q;
62 struct hlist_node *n;
63
64 hb = &f->hash[i];
65 hlist_for_each_entry_safe(q, n, &hb->chain, list) {
66 unsigned int hval = f->hashfn(q);
67
68 if (hval != i) {
69 struct inet_frag_bucket *hb_dest;
70
71 hlist_del(&q->list);
72
73
74 hb_dest = &f->hash[hval];
75 hlist_add_head(&q->list, &hb_dest->chain);
76 }
77 }
78 }
79 write_unlock(&f->lock);
80
81 mod_timer(&f->secret_timer, now + f->secret_interval);
82}
83
84void inet_frags_init(struct inet_frags *f)
85{
86 int i;
87
88 for (i = 0; i < INETFRAGS_HASHSZ; i++) {
89 struct inet_frag_bucket *hb = &f->hash[i];
90
91 spin_lock_init(&hb->chain_lock);
92 INIT_HLIST_HEAD(&hb->chain);
93 }
94 rwlock_init(&f->lock);
95
96 f->rnd = (u32) ((num_physpages ^ (num_physpages>>7)) ^
97 (jiffies ^ (jiffies >> 6)));
98
99 setup_timer(&f->secret_timer, inet_frag_secret_rebuild,
100 (unsigned long)f);
101 f->secret_timer.expires = jiffies + f->secret_interval;
102 add_timer(&f->secret_timer);
103}
104EXPORT_SYMBOL(inet_frags_init);
105
106void inet_frags_init_net(struct netns_frags *nf)
107{
108 nf->nqueues = 0;
109 init_frag_mem_limit(nf);
110 INIT_LIST_HEAD(&nf->lru_list);
111 spin_lock_init(&nf->lru_lock);
112}
113EXPORT_SYMBOL(inet_frags_init_net);
114
115void inet_frags_fini(struct inet_frags *f)
116{
117 del_timer(&f->secret_timer);
118}
119EXPORT_SYMBOL(inet_frags_fini);
120
121void inet_frags_exit_net(struct netns_frags *nf, struct inet_frags *f)
122{
123 nf->low_thresh = 0;
124
125 local_bh_disable();
126 inet_frag_evictor(nf, f, true);
127 local_bh_enable();
128
129 percpu_counter_destroy(&nf->mem);
130}
131EXPORT_SYMBOL(inet_frags_exit_net);
132
133static inline void fq_unlink(struct inet_frag_queue *fq, struct inet_frags *f)
134{
135 struct inet_frag_bucket *hb;
136 unsigned int hash;
137
138 read_lock(&f->lock);
139 hash = f->hashfn(fq);
140 hb = &f->hash[hash];
141
142 spin_lock(&hb->chain_lock);
143 hlist_del(&fq->list);
144 spin_unlock(&hb->chain_lock);
145
146 read_unlock(&f->lock);
147 inet_frag_lru_del(fq);
148}
149
150void inet_frag_kill(struct inet_frag_queue *fq, struct inet_frags *f)
151{
152 if (del_timer(&fq->timer))
153 atomic_dec(&fq->refcnt);
154
155 if (!(fq->last_in & INET_FRAG_COMPLETE)) {
156 fq_unlink(fq, f);
157 atomic_dec(&fq->refcnt);
158 fq->last_in |= INET_FRAG_COMPLETE;
159 }
160}
161EXPORT_SYMBOL(inet_frag_kill);
162
163static inline void frag_kfree_skb(struct netns_frags *nf, struct inet_frags *f,
164 struct sk_buff *skb)
165{
166 if (f->skb_free)
167 f->skb_free(skb);
168 kfree_skb(skb);
169}
170
171void inet_frag_destroy(struct inet_frag_queue *q, struct inet_frags *f,
172 int *work)
173{
174 struct sk_buff *fp;
175 struct netns_frags *nf;
176 unsigned int sum, sum_truesize = 0;
177
178 WARN_ON(!(q->last_in & INET_FRAG_COMPLETE));
179 WARN_ON(del_timer(&q->timer) != 0);
180
181
182 fp = q->fragments;
183 nf = q->net;
184 while (fp) {
185 struct sk_buff *xp = fp->next;
186
187 sum_truesize += fp->truesize;
188 frag_kfree_skb(nf, f, fp);
189 fp = xp;
190 }
191 sum = sum_truesize + f->qsize;
192 if (work)
193 *work -= sum;
194 sub_frag_mem_limit(q, sum);
195
196 if (f->destructor)
197 f->destructor(q);
198 kfree(q);
199
200}
201EXPORT_SYMBOL(inet_frag_destroy);
202
203int inet_frag_evictor(struct netns_frags *nf, struct inet_frags *f, bool force)
204{
205 struct inet_frag_queue *q;
206 int work, evicted = 0;
207
208 if (!force) {
209 if (frag_mem_limit(nf) <= nf->high_thresh)
210 return 0;
211 }
212
213 work = frag_mem_limit(nf) - nf->low_thresh;
214 while (work > 0) {
215 spin_lock(&nf->lru_lock);
216
217 if (list_empty(&nf->lru_list)) {
218 spin_unlock(&nf->lru_lock);
219 break;
220 }
221
222 q = list_first_entry(&nf->lru_list,
223 struct inet_frag_queue, lru_list);
224 atomic_inc(&q->refcnt);
225
226 list_del_init(&q->lru_list);
227
228 spin_unlock(&nf->lru_lock);
229
230 spin_lock(&q->lock);
231 if (!(q->last_in & INET_FRAG_COMPLETE))
232 inet_frag_kill(q, f);
233 spin_unlock(&q->lock);
234
235 if (atomic_dec_and_test(&q->refcnt))
236 inet_frag_destroy(q, f, &work);
237 evicted++;
238 }
239
240 return evicted;
241}
242EXPORT_SYMBOL(inet_frag_evictor);
243
244static struct inet_frag_queue *inet_frag_intern(struct netns_frags *nf,
245 struct inet_frag_queue *qp_in, struct inet_frags *f,
246 void *arg)
247{
248 struct inet_frag_bucket *hb;
249 struct inet_frag_queue *qp;
250#ifdef CONFIG_SMP
251#endif
252 unsigned int hash;
253
254 read_lock(&f->lock);
255
256
257
258
259
260 hash = f->hashfn(qp_in);
261 hb = &f->hash[hash];
262 spin_lock(&hb->chain_lock);
263
264#ifdef CONFIG_SMP
265
266
267
268
269 hlist_for_each_entry(qp, &hb->chain, list) {
270 if (qp->net == nf && f->match(qp, arg)) {
271 atomic_inc(&qp->refcnt);
272 spin_unlock(&hb->chain_lock);
273 read_unlock(&f->lock);
274 qp_in->last_in |= INET_FRAG_COMPLETE;
275 inet_frag_put(qp_in, f);
276 return qp;
277 }
278 }
279#endif
280 qp = qp_in;
281 if (!mod_timer(&qp->timer, jiffies + nf->timeout))
282 atomic_inc(&qp->refcnt);
283
284 atomic_inc(&qp->refcnt);
285 hlist_add_head(&qp->list, &hb->chain);
286 spin_unlock(&hb->chain_lock);
287 read_unlock(&f->lock);
288 inet_frag_lru_add(nf, qp);
289 return qp;
290}
291
292static struct inet_frag_queue *inet_frag_alloc(struct netns_frags *nf,
293 struct inet_frags *f, void *arg)
294{
295 struct inet_frag_queue *q;
296
297 q = kzalloc(f->qsize, GFP_ATOMIC);
298 if (q == NULL)
299 return NULL;
300
301 q->net = nf;
302 f->constructor(q, arg);
303 add_frag_mem_limit(q, f->qsize);
304
305 setup_timer(&q->timer, f->frag_expire, (unsigned long)q);
306 spin_lock_init(&q->lock);
307 atomic_set(&q->refcnt, 1);
308 INIT_LIST_HEAD(&q->lru_list);
309
310 return q;
311}
312
313static struct inet_frag_queue *inet_frag_create(struct netns_frags *nf,
314 struct inet_frags *f, void *arg)
315{
316 struct inet_frag_queue *q;
317
318 q = inet_frag_alloc(nf, f, arg);
319 if (q == NULL)
320 return NULL;
321
322 return inet_frag_intern(nf, q, f, arg);
323}
324
325struct inet_frag_queue *inet_frag_find(struct netns_frags *nf,
326 struct inet_frags *f, void *key, unsigned int hash)
327 __releases(&f->lock)
328{
329 struct inet_frag_bucket *hb;
330 struct inet_frag_queue *q;
331 int depth = 0;
332
333 hb = &f->hash[hash];
334
335 spin_lock(&hb->chain_lock);
336 hlist_for_each_entry(q, &hb->chain, list) {
337 if (q->net == nf && f->match(q, key)) {
338 atomic_inc(&q->refcnt);
339 spin_unlock(&hb->chain_lock);
340 read_unlock(&f->lock);
341 return q;
342 }
343 depth++;
344 }
345 spin_unlock(&hb->chain_lock);
346 read_unlock(&f->lock);
347
348 if (depth <= INETFRAGS_MAXDEPTH)
349 return inet_frag_create(nf, f, key);
350 else
351 return ERR_PTR(-ENOBUFS);
352}
353EXPORT_SYMBOL(inet_frag_find);
354
355void inet_frag_maybe_warn_overflow(struct inet_frag_queue *q,
356 const char *prefix)
357{
358 static const char msg[] = "inet_frag_find: Fragment hash bucket"
359 " list length grew over limit " __stringify(INETFRAGS_MAXDEPTH)
360 ". Dropping fragment.\n";
361
362 if (PTR_ERR(q) == -ENOBUFS)
363 LIMIT_NETDEBUG(KERN_WARNING "%s%s", prefix, msg);
364}
365EXPORT_SYMBOL(inet_frag_maybe_warn_overflow);
366
