1
2
3
4
5
6
7
8
9
10#include <linux/kernel.h>
11#include <linux/fs.h>
12#include <linux/gfp.h>
13#include <linux/mm.h>
14#include <linux/module.h>
15#include <linux/blkdev.h>
16#include <linux/backing-dev.h>
17#include <linux/task_io_accounting_ops.h>
18#include <linux/pagevec.h>
19#include <linux/pagemap.h>
20
21
22
23
24
25void
26file_ra_state_init(struct file_ra_state *ra, struct address_space *mapping)
27{
28 ra->ra_pages = mapping->backing_dev_info->ra_pages;
29 ra->prev_pos = -1;
30}
31EXPORT_SYMBOL_GPL(file_ra_state_init);
32
33#define list_to_page(head) (list_entry((head)->prev, struct page, lru))
34
35
36
37
38
39
40
41
42static void read_cache_pages_invalidate_page(struct address_space *mapping,
43 struct page *page)
44{
45 if (page_has_private(page)) {
46 if (!trylock_page(page))
47 BUG();
48 page->mapping = mapping;
49 do_invalidatepage(page, 0);
50 page->mapping = NULL;
51 unlock_page(page);
52 }
53 page_cache_release(page);
54}
55
56
57
58
59static void read_cache_pages_invalidate_pages(struct address_space *mapping,
60 struct list_head *pages)
61{
62 struct page *victim;
63
64 while (!list_empty(pages)) {
65 victim = list_to_page(pages);
66 list_del(&victim->lru);
67 read_cache_pages_invalidate_page(mapping, victim);
68 }
69}
70
71
72
73
74
75
76
77
78
79
80
81int read_cache_pages(struct address_space *mapping, struct list_head *pages,
82 int (*filler)(void *, struct page *), void *data)
83{
84 struct page *page;
85 int ret = 0;
86
87 while (!list_empty(pages)) {
88 page = list_to_page(pages);
89 list_del(&page->lru);
90 if (add_to_page_cache_lru(page, mapping,
91 page->index, GFP_KERNEL)) {
92 read_cache_pages_invalidate_page(mapping, page);
93 continue;
94 }
95 page_cache_release(page);
96
97 ret = filler(data, page);
98 if (unlikely(ret)) {
99 read_cache_pages_invalidate_pages(mapping, pages);
100 break;
101 }
102 task_io_account_read(PAGE_CACHE_SIZE);
103 }
104 return ret;
105}
106
107EXPORT_SYMBOL(read_cache_pages);
108
109static int read_pages(struct address_space *mapping, struct file *filp,
110 struct list_head *pages, unsigned nr_pages)
111{
112 unsigned page_idx;
113 int ret;
114
115 if (mapping->a_ops->readpages) {
116 ret = mapping->a_ops->readpages(filp, mapping, pages, nr_pages);
117
118 put_pages_list(pages);
119 goto out;
120 }
121
122 for (page_idx = 0; page_idx < nr_pages; page_idx++) {
123 struct page *page = list_to_page(pages);
124 list_del(&page->lru);
125 if (!add_to_page_cache_lru(page, mapping,
126 page->index, GFP_KERNEL)) {
127 mapping->a_ops->readpage(filp, page);
128 }
129 page_cache_release(page);
130 }
131 ret = 0;
132out:
133 return ret;
134}
135
136
137
138
139
140
141
142
143
144static int
145__do_page_cache_readahead(struct address_space *mapping, struct file *filp,
146 pgoff_t offset, unsigned long nr_to_read,
147 unsigned long lookahead_size)
148{
149 struct inode *inode = mapping->host;
150 struct page *page;
151 unsigned long end_index;
152 LIST_HEAD(page_pool);
153 int page_idx;
154 int ret = 0;
155 loff_t isize = i_size_read(inode);
156
157 if (isize == 0)
158 goto out;
159
160 end_index = ((isize - 1) >> PAGE_CACHE_SHIFT);
161
162
163
164
165 for (page_idx = 0; page_idx < nr_to_read; page_idx++) {
166 pgoff_t page_offset = offset + page_idx;
167
168 if (page_offset > end_index)
169 break;
170
171 rcu_read_lock();
172 page = radix_tree_lookup(&mapping->page_tree, page_offset);
173 rcu_read_unlock();
174 if (page)
175 continue;
176
177 page = page_cache_alloc_cold(mapping);
178 if (!page)
179 break;
180 page->index = page_offset;
181 list_add(&page->lru, &page_pool);
182 if (page_idx == nr_to_read - lookahead_size)
183 SetPageReadahead(page);
184 ret++;
185 }
186
187
188
189
190
191
192 if (ret)
193 read_pages(mapping, filp, &page_pool, ret);
194 BUG_ON(!list_empty(&page_pool));
195out:
196 return ret;
197}
198
199
200
201
202
203int force_page_cache_readahead(struct address_space *mapping, struct file *filp,
204 pgoff_t offset, unsigned long nr_to_read)
205{
206 int ret = 0;
207
208 if (unlikely(!mapping->a_ops->readpage && !mapping->a_ops->readpages))
209 return -EINVAL;
210
211 nr_to_read = max_sane_readahead(nr_to_read);
212 while (nr_to_read) {
213 int err;
214
215 unsigned long this_chunk = (2 * 1024 * 1024) / PAGE_CACHE_SIZE;
216
217 if (this_chunk > nr_to_read)
218 this_chunk = nr_to_read;
219 err = __do_page_cache_readahead(mapping, filp,
220 offset, this_chunk, 0);
221 if (err < 0) {
222 ret = err;
223 break;
224 }
225 ret += err;
226 offset += this_chunk;
227 nr_to_read -= this_chunk;
228 }
229 return ret;
230}
231
232
233
234
235
236unsigned long max_sane_readahead(unsigned long nr)
237{
238 return min(nr, (node_page_state(numa_node_id(), NR_INACTIVE_FILE)
239 + node_page_state(numa_node_id(), NR_FREE_PAGES)) / 2);
240}
241
242
243
244
245unsigned long ra_submit(struct file_ra_state *ra,
246 struct address_space *mapping, struct file *filp)
247{
248 int actual;
249
250 actual = __do_page_cache_readahead(mapping, filp,
251 ra->start, ra->size, ra->async_size);
252
253 return actual;
254}
255
256
257
258
259
260
261
262static unsigned long get_init_ra_size(unsigned long size, unsigned long max)
263{
264 unsigned long newsize = roundup_pow_of_two(size);
265
266 if (newsize <= max / 32)
267 newsize = newsize * 4;
268 else if (newsize <= max / 4)
269 newsize = newsize * 2;
270 else
271 newsize = max;
272
273 return newsize;
274}
275
276
277
278
279
280static unsigned long get_next_ra_size(struct file_ra_state *ra,
281 unsigned long max)
282{
283 unsigned long cur = ra->size;
284 unsigned long newsize;
285
286 if (cur < max / 16)
287 newsize = 4 * cur;
288 else
289 newsize = 2 * cur;
290
291 return min(newsize, max);
292}
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339static pgoff_t count_history_pages(struct address_space *mapping,
340 struct file_ra_state *ra,
341 pgoff_t offset, unsigned long max)
342{
343 pgoff_t head;
344
345 rcu_read_lock();
346 head = radix_tree_prev_hole(&mapping->page_tree, offset - 1, max);
347 rcu_read_unlock();
348
349 return offset - 1 - head;
350}
351
352
353
354
355static int try_context_readahead(struct address_space *mapping,
356 struct file_ra_state *ra,
357 pgoff_t offset,
358 unsigned long req_size,
359 unsigned long max)
360{
361 pgoff_t size;
362
363 size = count_history_pages(mapping, ra, offset, max);
364
365
366
367
368
369 if (!size)
370 return 0;
371
372
373
374
375
376 if (size >= offset)
377 size *= 2;
378
379 ra->start = offset;
380 ra->size = get_init_ra_size(size + req_size, max);
381 ra->async_size = ra->size;
382
383 return 1;
384}
385
386
387
388
389static unsigned long
390ondemand_readahead(struct address_space *mapping,
391 struct file_ra_state *ra, struct file *filp,
392 bool hit_readahead_marker, pgoff_t offset,
393 unsigned long req_size)
394{
395 unsigned long max = max_sane_readahead(ra->ra_pages);
396
397
398
399
400 if (!offset)
401 goto initial_readahead;
402
403
404
405
406
407 if ((offset == (ra->start + ra->size - ra->async_size) ||
408 offset == (ra->start + ra->size))) {
409 ra->start += ra->size;
410 ra->size = get_next_ra_size(ra, max);
411 ra->async_size = ra->size;
412 goto readit;
413 }
414
415
416
417
418
419
420
421 if (hit_readahead_marker) {
422 pgoff_t start;
423
424 rcu_read_lock();
425 start = radix_tree_next_hole(&mapping->page_tree, offset+1,max);
426 rcu_read_unlock();
427
428 if (!start || start - offset > max)
429 return 0;
430
431 ra->start = start;
432 ra->size = start - offset;
433 ra->size += req_size;
434 ra->size = get_next_ra_size(ra, max);
435 ra->async_size = ra->size;
436 goto readit;
437 }
438
439
440
441
442 if (req_size > max)
443 goto initial_readahead;
444
445
446
447
448 if (offset - (ra->prev_pos >> PAGE_CACHE_SHIFT) <= 1UL)
449 goto initial_readahead;
450
451
452
453
454
455 if (try_context_readahead(mapping, ra, offset, req_size, max))
456 goto readit;
457
458
459
460
461
462 return __do_page_cache_readahead(mapping, filp, offset, req_size, 0);
463
464initial_readahead:
465 ra->start = offset;
466 ra->size = get_init_ra_size(req_size, max);
467 ra->async_size = ra->size > req_size ? ra->size - req_size : ra->size;
468
469readit:
470
471
472
473
474
475 if (offset == ra->start && ra->size == ra->async_size) {
476 ra->async_size = get_next_ra_size(ra, max);
477 ra->size += ra->async_size;
478 }
479
480 return ra_submit(ra, mapping, filp);
481}
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497void page_cache_sync_readahead(struct address_space *mapping,
498 struct file_ra_state *ra, struct file *filp,
499 pgoff_t offset, unsigned long req_size)
500{
501
502 if (!ra->ra_pages)
503 return;
504
505
506 if (filp && (filp->f_mode & FMODE_RANDOM)) {
507 force_page_cache_readahead(mapping, filp, offset, req_size);
508 return;
509 }
510
511
512 ondemand_readahead(mapping, ra, filp, false, offset, req_size);
513}
514EXPORT_SYMBOL_GPL(page_cache_sync_readahead);
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531void
532page_cache_async_readahead(struct address_space *mapping,
533 struct file_ra_state *ra, struct file *filp,
534 struct page *page, pgoff_t offset,
535 unsigned long req_size)
536{
537
538 if (!ra->ra_pages)
539 return;
540
541
542
543
544 if (PageWriteback(page))
545 return;
546
547 ClearPageReadahead(page);
548
549
550
551
552 if (bdi_read_congested(mapping->backing_dev_info))
553 return;
554
555
556 ondemand_readahead(mapping, ra, filp, true, offset, req_size);
557
558#ifdef CONFIG_BLOCK
559
560
561
562
563
564
565
566 if (PageUptodate(page))
567 blk_run_backing_dev(mapping->backing_dev_info, NULL);
568#endif
569}
570EXPORT_SYMBOL_GPL(page_cache_async_readahead);
571