1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16#include <linux/kernel.h>
17#include <linux/export.h>
18#include <linux/mm.h>
19#include <linux/kdev_t.h>
20#include <linux/gfp.h>
21#include <linux/bio.h>
22#include <linux/fs.h>
23#include <linux/buffer_head.h>
24#include <linux/blkdev.h>
25#include <linux/highmem.h>
26#include <linux/prefetch.h>
27#include <linux/mpage.h>
28#include <linux/mm_inline.h>
29#include <linux/writeback.h>
30#include <linux/backing-dev.h>
31#include <linux/pagevec.h>
32#include <linux/cleancache.h>
33#include "internal.h"
34
35
36
37
38
39
40
41
42
43
44
45
46
47static void mpage_end_io(struct bio *bio)
48{
49 struct bio_vec *bv;
50 struct bvec_iter_all iter_all;
51
52 bio_for_each_segment_all(bv, bio, iter_all) {
53 struct page *page = bv->bv_page;
54 page_endio(page, bio_op(bio),
55 blk_status_to_errno(bio->bi_status));
56 }
57
58 bio_put(bio);
59}
60
61static struct bio *mpage_bio_submit(int op, int op_flags, struct bio *bio)
62{
63 bio->bi_end_io = mpage_end_io;
64 bio_set_op_attrs(bio, op, op_flags);
65 guard_bio_eod(bio);
66 submit_bio(bio);
67 return NULL;
68}
69
70static struct bio *
71mpage_alloc(struct block_device *bdev,
72 sector_t first_sector, int nr_vecs,
73 gfp_t gfp_flags)
74{
75 struct bio *bio;
76
77
78 gfp_flags &= GFP_KERNEL;
79 bio = bio_alloc(gfp_flags, nr_vecs);
80
81 if (bio == NULL && (current->flags & PF_MEMALLOC)) {
82 while (!bio && (nr_vecs /= 2))
83 bio = bio_alloc(gfp_flags, nr_vecs);
84 }
85
86 if (bio) {
87 bio_set_dev(bio, bdev);
88 bio->bi_iter.bi_sector = first_sector;
89 }
90 return bio;
91}
92
93
94
95
96
97
98
99
100
101
102
103static void
104map_buffer_to_page(struct page *page, struct buffer_head *bh, int page_block)
105{
106 struct inode *inode = page->mapping->host;
107 struct buffer_head *page_bh, *head;
108 int block = 0;
109
110 if (!page_has_buffers(page)) {
111
112
113
114
115 if (inode->i_blkbits == PAGE_SHIFT &&
116 buffer_uptodate(bh)) {
117 SetPageUptodate(page);
118 return;
119 }
120 create_empty_buffers(page, i_blocksize(inode), 0);
121 }
122 head = page_buffers(page);
123 page_bh = head;
124 do {
125 if (block == page_block) {
126 page_bh->b_state = bh->b_state;
127 page_bh->b_bdev = bh->b_bdev;
128 page_bh->b_blocknr = bh->b_blocknr;
129 break;
130 }
131 page_bh = page_bh->b_this_page;
132 block++;
133 } while (page_bh != head);
134}
135
136struct mpage_readpage_args {
137 struct bio *bio;
138 struct page *page;
139 unsigned int nr_pages;
140 bool is_readahead;
141 sector_t last_block_in_bio;
142 struct buffer_head map_bh;
143 unsigned long first_logical_block;
144 get_block_t *get_block;
145};
146
147
148
149
150
151
152
153
154
155
156static struct bio *do_mpage_readpage(struct mpage_readpage_args *args)
157{
158 struct page *page = args->page;
159 struct inode *inode = page->mapping->host;
160 const unsigned blkbits = inode->i_blkbits;
161 const unsigned blocks_per_page = PAGE_SIZE >> blkbits;
162 const unsigned blocksize = 1 << blkbits;
163 struct buffer_head *map_bh = &args->map_bh;
164 sector_t block_in_file;
165 sector_t last_block;
166 sector_t last_block_in_file;
167 sector_t blocks[MAX_BUF_PER_PAGE];
168 unsigned page_block;
169 unsigned first_hole = blocks_per_page;
170 struct block_device *bdev = NULL;
171 int length;
172 int fully_mapped = 1;
173 int op_flags;
174 unsigned nblocks;
175 unsigned relative_block;
176 gfp_t gfp;
177
178 if (args->is_readahead) {
179 op_flags = REQ_RAHEAD;
180 gfp = readahead_gfp_mask(page->mapping);
181 } else {
182 op_flags = 0;
183 gfp = mapping_gfp_constraint(page->mapping, GFP_KERNEL);
184 }
185
186 if (page_has_buffers(page))
187 goto confused;
188
189 block_in_file = (sector_t)page->index << (PAGE_SHIFT - blkbits);
190 last_block = block_in_file + args->nr_pages * blocks_per_page;
191 last_block_in_file = (i_size_read(inode) + blocksize - 1) >> blkbits;
192 if (last_block > last_block_in_file)
193 last_block = last_block_in_file;
194 page_block = 0;
195
196
197
198
199 nblocks = map_bh->b_size >> blkbits;
200 if (buffer_mapped(map_bh) &&
201 block_in_file > args->first_logical_block &&
202 block_in_file < (args->first_logical_block + nblocks)) {
203 unsigned map_offset = block_in_file - args->first_logical_block;
204 unsigned last = nblocks - map_offset;
205
206 for (relative_block = 0; ; relative_block++) {
207 if (relative_block == last) {
208 clear_buffer_mapped(map_bh);
209 break;
210 }
211 if (page_block == blocks_per_page)
212 break;
213 blocks[page_block] = map_bh->b_blocknr + map_offset +
214 relative_block;
215 page_block++;
216 block_in_file++;
217 }
218 bdev = map_bh->b_bdev;
219 }
220
221
222
223
224 map_bh->b_page = page;
225 while (page_block < blocks_per_page) {
226 map_bh->b_state = 0;
227 map_bh->b_size = 0;
228
229 if (block_in_file < last_block) {
230 map_bh->b_size = (last_block-block_in_file) << blkbits;
231 if (args->get_block(inode, block_in_file, map_bh, 0))
232 goto confused;
233 args->first_logical_block = block_in_file;
234 }
235
236 if (!buffer_mapped(map_bh)) {
237 fully_mapped = 0;
238 if (first_hole == blocks_per_page)
239 first_hole = page_block;
240 page_block++;
241 block_in_file++;
242 continue;
243 }
244
245
246
247
248
249
250
251 if (buffer_uptodate(map_bh)) {
252 map_buffer_to_page(page, map_bh, page_block);
253 goto confused;
254 }
255
256 if (first_hole != blocks_per_page)
257 goto confused;
258
259
260 if (page_block && blocks[page_block-1] != map_bh->b_blocknr-1)
261 goto confused;
262 nblocks = map_bh->b_size >> blkbits;
263 for (relative_block = 0; ; relative_block++) {
264 if (relative_block == nblocks) {
265 clear_buffer_mapped(map_bh);
266 break;
267 } else if (page_block == blocks_per_page)
268 break;
269 blocks[page_block] = map_bh->b_blocknr+relative_block;
270 page_block++;
271 block_in_file++;
272 }
273 bdev = map_bh->b_bdev;
274 }
275
276 if (first_hole != blocks_per_page) {
277 zero_user_segment(page, first_hole << blkbits, PAGE_SIZE);
278 if (first_hole == 0) {
279 SetPageUptodate(page);
280 unlock_page(page);
281 goto out;
282 }
283 } else if (fully_mapped) {
284 SetPageMappedToDisk(page);
285 }
286
287 if (fully_mapped && blocks_per_page == 1 && !PageUptodate(page) &&
288 cleancache_get_page(page) == 0) {
289 SetPageUptodate(page);
290 goto confused;
291 }
292
293
294
295
296 if (args->bio && (args->last_block_in_bio != blocks[0] - 1))
297 args->bio = mpage_bio_submit(REQ_OP_READ, op_flags, args->bio);
298
299alloc_new:
300 if (args->bio == NULL) {
301 if (first_hole == blocks_per_page) {
302 if (!bdev_read_page(bdev, blocks[0] << (blkbits - 9),
303 page))
304 goto out;
305 }
306 args->bio = mpage_alloc(bdev, blocks[0] << (blkbits - 9),
307 bio_max_segs(args->nr_pages), gfp);
308 if (args->bio == NULL)
309 goto confused;
310 }
311
312 length = first_hole << blkbits;
313 if (bio_add_page(args->bio, page, length, 0) < length) {
314 args->bio = mpage_bio_submit(REQ_OP_READ, op_flags, args->bio);
315 goto alloc_new;
316 }
317
318 relative_block = block_in_file - args->first_logical_block;
319 nblocks = map_bh->b_size >> blkbits;
320 if ((buffer_boundary(map_bh) && relative_block == nblocks) ||
321 (first_hole != blocks_per_page))
322 args->bio = mpage_bio_submit(REQ_OP_READ, op_flags, args->bio);
323 else
324 args->last_block_in_bio = blocks[blocks_per_page - 1];
325out:
326 return args->bio;
327
328confused:
329 if (args->bio)
330 args->bio = mpage_bio_submit(REQ_OP_READ, op_flags, args->bio);
331 if (!PageUptodate(page))
332 block_read_full_page(page, args->get_block);
333 else
334 unlock_page(page);
335 goto out;
336}
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377void mpage_readahead(struct readahead_control *rac, get_block_t get_block)
378{
379 struct page *page;
380 struct mpage_readpage_args args = {
381 .get_block = get_block,
382 .is_readahead = true,
383 };
384
385 while ((page = readahead_page(rac))) {
386 prefetchw(&page->flags);
387 args.page = page;
388 args.nr_pages = readahead_count(rac);
389 args.bio = do_mpage_readpage(&args);
390 put_page(page);
391 }
392 if (args.bio)
393 mpage_bio_submit(REQ_OP_READ, REQ_RAHEAD, args.bio);
394}
395EXPORT_SYMBOL(mpage_readahead);
396
397
398
399
400int mpage_readpage(struct page *page, get_block_t get_block)
401{
402 struct mpage_readpage_args args = {
403 .page = page,
404 .nr_pages = 1,
405 .get_block = get_block,
406 };
407
408 args.bio = do_mpage_readpage(&args);
409 if (args.bio)
410 mpage_bio_submit(REQ_OP_READ, 0, args.bio);
411 return 0;
412}
413EXPORT_SYMBOL(mpage_readpage);
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432struct mpage_data {
433 struct bio *bio;
434 sector_t last_block_in_bio;
435 get_block_t *get_block;
436 unsigned use_writepage;
437};
438
439
440
441
442
443static void clean_buffers(struct page *page, unsigned first_unmapped)
444{
445 unsigned buffer_counter = 0;
446 struct buffer_head *bh, *head;
447 if (!page_has_buffers(page))
448 return;
449 head = page_buffers(page);
450 bh = head;
451
452 do {
453 if (buffer_counter++ == first_unmapped)
454 break;
455 clear_buffer_dirty(bh);
456 bh = bh->b_this_page;
457 } while (bh != head);
458
459
460
461
462
463
464 if (buffer_heads_over_limit && PageUptodate(page))
465 try_to_free_buffers(page);
466}
467
468
469
470
471
472
473void clean_page_buffers(struct page *page)
474{
475 clean_buffers(page, ~0U);
476}
477
478static int __mpage_writepage(struct page *page, struct writeback_control *wbc,
479 void *data)
480{
481 struct mpage_data *mpd = data;
482 struct bio *bio = mpd->bio;
483 struct address_space *mapping = page->mapping;
484 struct inode *inode = page->mapping->host;
485 const unsigned blkbits = inode->i_blkbits;
486 unsigned long end_index;
487 const unsigned blocks_per_page = PAGE_SIZE >> blkbits;
488 sector_t last_block;
489 sector_t block_in_file;
490 sector_t blocks[MAX_BUF_PER_PAGE];
491 unsigned page_block;
492 unsigned first_unmapped = blocks_per_page;
493 struct block_device *bdev = NULL;
494 int boundary = 0;
495 sector_t boundary_block = 0;
496 struct block_device *boundary_bdev = NULL;
497 int length;
498 struct buffer_head map_bh;
499 loff_t i_size = i_size_read(inode);
500 int ret = 0;
501 int op_flags = wbc_to_write_flags(wbc);
502
503 if (page_has_buffers(page)) {
504 struct buffer_head *head = page_buffers(page);
505 struct buffer_head *bh = head;
506
507
508 page_block = 0;
509 do {
510 BUG_ON(buffer_locked(bh));
511 if (!buffer_mapped(bh)) {
512
513
514
515
516 if (buffer_dirty(bh))
517 goto confused;
518 if (first_unmapped == blocks_per_page)
519 first_unmapped = page_block;
520 continue;
521 }
522
523 if (first_unmapped != blocks_per_page)
524 goto confused;
525
526 if (!buffer_dirty(bh) || !buffer_uptodate(bh))
527 goto confused;
528 if (page_block) {
529 if (bh->b_blocknr != blocks[page_block-1] + 1)
530 goto confused;
531 }
532 blocks[page_block++] = bh->b_blocknr;
533 boundary = buffer_boundary(bh);
534 if (boundary) {
535 boundary_block = bh->b_blocknr;
536 boundary_bdev = bh->b_bdev;
537 }
538 bdev = bh->b_bdev;
539 } while ((bh = bh->b_this_page) != head);
540
541 if (first_unmapped)
542 goto page_is_mapped;
543
544
545
546
547
548
549
550 goto confused;
551 }
552
553
554
555
556 BUG_ON(!PageUptodate(page));
557 block_in_file = (sector_t)page->index << (PAGE_SHIFT - blkbits);
558 last_block = (i_size - 1) >> blkbits;
559 map_bh.b_page = page;
560 for (page_block = 0; page_block < blocks_per_page; ) {
561
562 map_bh.b_state = 0;
563 map_bh.b_size = 1 << blkbits;
564 if (mpd->get_block(inode, block_in_file, &map_bh, 1))
565 goto confused;
566 if (buffer_new(&map_bh))
567 clean_bdev_bh_alias(&map_bh);
568 if (buffer_boundary(&map_bh)) {
569 boundary_block = map_bh.b_blocknr;
570 boundary_bdev = map_bh.b_bdev;
571 }
572 if (page_block) {
573 if (map_bh.b_blocknr != blocks[page_block-1] + 1)
574 goto confused;
575 }
576 blocks[page_block++] = map_bh.b_blocknr;
577 boundary = buffer_boundary(&map_bh);
578 bdev = map_bh.b_bdev;
579 if (block_in_file == last_block)
580 break;
581 block_in_file++;
582 }
583 BUG_ON(page_block == 0);
584
585 first_unmapped = page_block;
586
587page_is_mapped:
588 end_index = i_size >> PAGE_SHIFT;
589 if (page->index >= end_index) {
590
591
592
593
594
595
596
597
598 unsigned offset = i_size & (PAGE_SIZE - 1);
599
600 if (page->index > end_index || !offset)
601 goto confused;
602 zero_user_segment(page, offset, PAGE_SIZE);
603 }
604
605
606
607
608 if (bio && mpd->last_block_in_bio != blocks[0] - 1)
609 bio = mpage_bio_submit(REQ_OP_WRITE, op_flags, bio);
610
611alloc_new:
612 if (bio == NULL) {
613 if (first_unmapped == blocks_per_page) {
614 if (!bdev_write_page(bdev, blocks[0] << (blkbits - 9),
615 page, wbc))
616 goto out;
617 }
618 bio = mpage_alloc(bdev, blocks[0] << (blkbits - 9),
619 BIO_MAX_VECS, GFP_NOFS|__GFP_HIGH);
620 if (bio == NULL)
621 goto confused;
622
623 wbc_init_bio(wbc, bio);
624 bio->bi_write_hint = inode->i_write_hint;
625 }
626
627
628
629
630
631
632 wbc_account_cgroup_owner(wbc, page, PAGE_SIZE);
633 length = first_unmapped << blkbits;
634 if (bio_add_page(bio, page, length, 0) < length) {
635 bio = mpage_bio_submit(REQ_OP_WRITE, op_flags, bio);
636 goto alloc_new;
637 }
638
639 clean_buffers(page, first_unmapped);
640
641 BUG_ON(PageWriteback(page));
642 set_page_writeback(page);
643 unlock_page(page);
644 if (boundary || (first_unmapped != blocks_per_page)) {
645 bio = mpage_bio_submit(REQ_OP_WRITE, op_flags, bio);
646 if (boundary_block) {
647 write_boundary_block(boundary_bdev,
648 boundary_block, 1 << blkbits);
649 }
650 } else {
651 mpd->last_block_in_bio = blocks[blocks_per_page - 1];
652 }
653 goto out;
654
655confused:
656 if (bio)
657 bio = mpage_bio_submit(REQ_OP_WRITE, op_flags, bio);
658
659 if (mpd->use_writepage) {
660 ret = mapping->a_ops->writepage(page, wbc);
661 } else {
662 ret = -EAGAIN;
663 goto out;
664 }
665
666
667
668 mapping_set_error(mapping, ret);
669out:
670 mpd->bio = bio;
671 return ret;
672}
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693int
694mpage_writepages(struct address_space *mapping,
695 struct writeback_control *wbc, get_block_t get_block)
696{
697 struct blk_plug plug;
698 int ret;
699
700 blk_start_plug(&plug);
701
702 if (!get_block)
703 ret = generic_writepages(mapping, wbc);
704 else {
705 struct mpage_data mpd = {
706 .bio = NULL,
707 .last_block_in_bio = 0,
708 .get_block = get_block,
709 .use_writepage = 1,
710 };
711
712 ret = write_cache_pages(mapping, wbc, __mpage_writepage, &mpd);
713 if (mpd.bio) {
714 int op_flags = (wbc->sync_mode == WB_SYNC_ALL ?
715 REQ_SYNC : 0);
716 mpage_bio_submit(REQ_OP_WRITE, op_flags, mpd.bio);
717 }
718 }
719 blk_finish_plug(&plug);
720 return ret;
721}
722EXPORT_SYMBOL(mpage_writepages);
723
724int mpage_writepage(struct page *page, get_block_t get_block,
725 struct writeback_control *wbc)
726{
727 struct mpage_data mpd = {
728 .bio = NULL,
729 .last_block_in_bio = 0,
730 .get_block = get_block,
731 .use_writepage = 0,
732 };
733 int ret = __mpage_writepage(page, wbc, &mpd);
734 if (mpd.bio) {
735 int op_flags = (wbc->sync_mode == WB_SYNC_ALL ?
736 REQ_SYNC : 0);
737 mpage_bio_submit(REQ_OP_WRITE, op_flags, mpd.bio);
738 }
739 return ret;
740}
741EXPORT_SYMBOL(mpage_writepage);
742