1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16#include <linux/kernel.h>
17#include <linux/export.h>
18#include <linux/mm.h>
19#include <linux/kdev_t.h>
20#include <linux/gfp.h>
21#include <linux/bio.h>
22#include <linux/fs.h>
23#include <linux/buffer_head.h>
24#include <linux/blkdev.h>
25#include <linux/highmem.h>
26#include <linux/prefetch.h>
27#include <linux/mpage.h>
28#include <linux/mm_inline.h>
29#include <linux/writeback.h>
30#include <linux/backing-dev.h>
31#include <linux/pagevec.h>
32#include "internal.h"
33
34
35
36
37
38
39
40
41
42
43
44
45
46static void mpage_end_io(struct bio *bio)
47{
48 struct bio_vec *bv;
49 struct bvec_iter_all iter_all;
50
51 bio_for_each_segment_all(bv, bio, iter_all) {
52 struct page *page = bv->bv_page;
53 page_endio(page, bio_op(bio),
54 blk_status_to_errno(bio->bi_status));
55 }
56
57 bio_put(bio);
58}
59
60static struct bio *mpage_bio_submit(int op, int op_flags, struct bio *bio)
61{
62 bio->bi_end_io = mpage_end_io;
63 bio_set_op_attrs(bio, op, op_flags);
64 guard_bio_eod(bio);
65 submit_bio(bio);
66 return NULL;
67}
68
69static struct bio *
70mpage_alloc(struct block_device *bdev,
71 sector_t first_sector, int nr_vecs,
72 gfp_t gfp_flags)
73{
74 struct bio *bio;
75
76
77 gfp_flags &= GFP_KERNEL;
78 bio = bio_alloc(gfp_flags, nr_vecs);
79
80 if (bio == NULL && (current->flags & PF_MEMALLOC)) {
81 while (!bio && (nr_vecs /= 2))
82 bio = bio_alloc(gfp_flags, nr_vecs);
83 }
84
85 if (bio) {
86 bio_set_dev(bio, bdev);
87 bio->bi_iter.bi_sector = first_sector;
88 }
89 return bio;
90}
91
92
93
94
95
96
97
98
99
100
101
102static void
103map_buffer_to_page(struct page *page, struct buffer_head *bh, int page_block)
104{
105 struct inode *inode = page->mapping->host;
106 struct buffer_head *page_bh, *head;
107 int block = 0;
108
109 if (!page_has_buffers(page)) {
110
111
112
113
114 if (inode->i_blkbits == PAGE_SHIFT &&
115 buffer_uptodate(bh)) {
116 SetPageUptodate(page);
117 return;
118 }
119 create_empty_buffers(page, i_blocksize(inode), 0);
120 }
121 head = page_buffers(page);
122 page_bh = head;
123 do {
124 if (block == page_block) {
125 page_bh->b_state = bh->b_state;
126 page_bh->b_bdev = bh->b_bdev;
127 page_bh->b_blocknr = bh->b_blocknr;
128 break;
129 }
130 page_bh = page_bh->b_this_page;
131 block++;
132 } while (page_bh != head);
133}
134
135struct mpage_readpage_args {
136 struct bio *bio;
137 struct page *page;
138 unsigned int nr_pages;
139 bool is_readahead;
140 sector_t last_block_in_bio;
141 struct buffer_head map_bh;
142 unsigned long first_logical_block;
143 get_block_t *get_block;
144};
145
146
147
148
149
150
151
152
153
154
155static struct bio *do_mpage_readpage(struct mpage_readpage_args *args)
156{
157 struct page *page = args->page;
158 struct inode *inode = page->mapping->host;
159 const unsigned blkbits = inode->i_blkbits;
160 const unsigned blocks_per_page = PAGE_SIZE >> blkbits;
161 const unsigned blocksize = 1 << blkbits;
162 struct buffer_head *map_bh = &args->map_bh;
163 sector_t block_in_file;
164 sector_t last_block;
165 sector_t last_block_in_file;
166 sector_t blocks[MAX_BUF_PER_PAGE];
167 unsigned page_block;
168 unsigned first_hole = blocks_per_page;
169 struct block_device *bdev = NULL;
170 int length;
171 int fully_mapped = 1;
172 int op_flags;
173 unsigned nblocks;
174 unsigned relative_block;
175 gfp_t gfp;
176
177 if (args->is_readahead) {
178 op_flags = REQ_RAHEAD;
179 gfp = readahead_gfp_mask(page->mapping);
180 } else {
181 op_flags = 0;
182 gfp = mapping_gfp_constraint(page->mapping, GFP_KERNEL);
183 }
184
185 if (page_has_buffers(page))
186 goto confused;
187
188 block_in_file = (sector_t)page->index << (PAGE_SHIFT - blkbits);
189 last_block = block_in_file + args->nr_pages * blocks_per_page;
190 last_block_in_file = (i_size_read(inode) + blocksize - 1) >> blkbits;
191 if (last_block > last_block_in_file)
192 last_block = last_block_in_file;
193 page_block = 0;
194
195
196
197
198 nblocks = map_bh->b_size >> blkbits;
199 if (buffer_mapped(map_bh) &&
200 block_in_file > args->first_logical_block &&
201 block_in_file < (args->first_logical_block + nblocks)) {
202 unsigned map_offset = block_in_file - args->first_logical_block;
203 unsigned last = nblocks - map_offset;
204
205 for (relative_block = 0; ; relative_block++) {
206 if (relative_block == last) {
207 clear_buffer_mapped(map_bh);
208 break;
209 }
210 if (page_block == blocks_per_page)
211 break;
212 blocks[page_block] = map_bh->b_blocknr + map_offset +
213 relative_block;
214 page_block++;
215 block_in_file++;
216 }
217 bdev = map_bh->b_bdev;
218 }
219
220
221
222
223 map_bh->b_page = page;
224 while (page_block < blocks_per_page) {
225 map_bh->b_state = 0;
226 map_bh->b_size = 0;
227
228 if (block_in_file < last_block) {
229 map_bh->b_size = (last_block-block_in_file) << blkbits;
230 if (args->get_block(inode, block_in_file, map_bh, 0))
231 goto confused;
232 args->first_logical_block = block_in_file;
233 }
234
235 if (!buffer_mapped(map_bh)) {
236 fully_mapped = 0;
237 if (first_hole == blocks_per_page)
238 first_hole = page_block;
239 page_block++;
240 block_in_file++;
241 continue;
242 }
243
244
245
246
247
248
249
250 if (buffer_uptodate(map_bh)) {
251 map_buffer_to_page(page, map_bh, page_block);
252 goto confused;
253 }
254
255 if (first_hole != blocks_per_page)
256 goto confused;
257
258
259 if (page_block && blocks[page_block-1] != map_bh->b_blocknr-1)
260 goto confused;
261 nblocks = map_bh->b_size >> blkbits;
262 for (relative_block = 0; ; relative_block++) {
263 if (relative_block == nblocks) {
264 clear_buffer_mapped(map_bh);
265 break;
266 } else if (page_block == blocks_per_page)
267 break;
268 blocks[page_block] = map_bh->b_blocknr+relative_block;
269 page_block++;
270 block_in_file++;
271 }
272 bdev = map_bh->b_bdev;
273 }
274
275 if (first_hole != blocks_per_page) {
276 zero_user_segment(page, first_hole << blkbits, PAGE_SIZE);
277 if (first_hole == 0) {
278 SetPageUptodate(page);
279 unlock_page(page);
280 goto out;
281 }
282 } else if (fully_mapped) {
283 SetPageMappedToDisk(page);
284 }
285
286
287
288
289 if (args->bio && (args->last_block_in_bio != blocks[0] - 1))
290 args->bio = mpage_bio_submit(REQ_OP_READ, op_flags, args->bio);
291
292alloc_new:
293 if (args->bio == NULL) {
294 if (first_hole == blocks_per_page) {
295 if (!bdev_read_page(bdev, blocks[0] << (blkbits - 9),
296 page))
297 goto out;
298 }
299 args->bio = mpage_alloc(bdev, blocks[0] << (blkbits - 9),
300 bio_max_segs(args->nr_pages), gfp);
301 if (args->bio == NULL)
302 goto confused;
303 }
304
305 length = first_hole << blkbits;
306 if (bio_add_page(args->bio, page, length, 0) < length) {
307 args->bio = mpage_bio_submit(REQ_OP_READ, op_flags, args->bio);
308 goto alloc_new;
309 }
310
311 relative_block = block_in_file - args->first_logical_block;
312 nblocks = map_bh->b_size >> blkbits;
313 if ((buffer_boundary(map_bh) && relative_block == nblocks) ||
314 (first_hole != blocks_per_page))
315 args->bio = mpage_bio_submit(REQ_OP_READ, op_flags, args->bio);
316 else
317 args->last_block_in_bio = blocks[blocks_per_page - 1];
318out:
319 return args->bio;
320
321confused:
322 if (args->bio)
323 args->bio = mpage_bio_submit(REQ_OP_READ, op_flags, args->bio);
324 if (!PageUptodate(page))
325 block_read_full_page(page, args->get_block);
326 else
327 unlock_page(page);
328 goto out;
329}
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370void mpage_readahead(struct readahead_control *rac, get_block_t get_block)
371{
372 struct page *page;
373 struct mpage_readpage_args args = {
374 .get_block = get_block,
375 .is_readahead = true,
376 };
377
378 while ((page = readahead_page(rac))) {
379 prefetchw(&page->flags);
380 args.page = page;
381 args.nr_pages = readahead_count(rac);
382 args.bio = do_mpage_readpage(&args);
383 put_page(page);
384 }
385 if (args.bio)
386 mpage_bio_submit(REQ_OP_READ, REQ_RAHEAD, args.bio);
387}
388EXPORT_SYMBOL(mpage_readahead);
389
390
391
392
393int mpage_readpage(struct page *page, get_block_t get_block)
394{
395 struct mpage_readpage_args args = {
396 .page = page,
397 .nr_pages = 1,
398 .get_block = get_block,
399 };
400
401 args.bio = do_mpage_readpage(&args);
402 if (args.bio)
403 mpage_bio_submit(REQ_OP_READ, 0, args.bio);
404 return 0;
405}
406EXPORT_SYMBOL(mpage_readpage);
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425struct mpage_data {
426 struct bio *bio;
427 sector_t last_block_in_bio;
428 get_block_t *get_block;
429 unsigned use_writepage;
430};
431
432
433
434
435
436static void clean_buffers(struct page *page, unsigned first_unmapped)
437{
438 unsigned buffer_counter = 0;
439 struct buffer_head *bh, *head;
440 if (!page_has_buffers(page))
441 return;
442 head = page_buffers(page);
443 bh = head;
444
445 do {
446 if (buffer_counter++ == first_unmapped)
447 break;
448 clear_buffer_dirty(bh);
449 bh = bh->b_this_page;
450 } while (bh != head);
451
452
453
454
455
456
457 if (buffer_heads_over_limit && PageUptodate(page))
458 try_to_free_buffers(page);
459}
460
461
462
463
464
465
466void clean_page_buffers(struct page *page)
467{
468 clean_buffers(page, ~0U);
469}
470
471static int __mpage_writepage(struct page *page, struct writeback_control *wbc,
472 void *data)
473{
474 struct mpage_data *mpd = data;
475 struct bio *bio = mpd->bio;
476 struct address_space *mapping = page->mapping;
477 struct inode *inode = page->mapping->host;
478 const unsigned blkbits = inode->i_blkbits;
479 unsigned long end_index;
480 const unsigned blocks_per_page = PAGE_SIZE >> blkbits;
481 sector_t last_block;
482 sector_t block_in_file;
483 sector_t blocks[MAX_BUF_PER_PAGE];
484 unsigned page_block;
485 unsigned first_unmapped = blocks_per_page;
486 struct block_device *bdev = NULL;
487 int boundary = 0;
488 sector_t boundary_block = 0;
489 struct block_device *boundary_bdev = NULL;
490 int length;
491 struct buffer_head map_bh;
492 loff_t i_size = i_size_read(inode);
493 int ret = 0;
494 int op_flags = wbc_to_write_flags(wbc);
495
496 if (page_has_buffers(page)) {
497 struct buffer_head *head = page_buffers(page);
498 struct buffer_head *bh = head;
499
500
501 page_block = 0;
502 do {
503 BUG_ON(buffer_locked(bh));
504 if (!buffer_mapped(bh)) {
505
506
507
508
509 if (buffer_dirty(bh))
510 goto confused;
511 if (first_unmapped == blocks_per_page)
512 first_unmapped = page_block;
513 continue;
514 }
515
516 if (first_unmapped != blocks_per_page)
517 goto confused;
518
519 if (!buffer_dirty(bh) || !buffer_uptodate(bh))
520 goto confused;
521 if (page_block) {
522 if (bh->b_blocknr != blocks[page_block-1] + 1)
523 goto confused;
524 }
525 blocks[page_block++] = bh->b_blocknr;
526 boundary = buffer_boundary(bh);
527 if (boundary) {
528 boundary_block = bh->b_blocknr;
529 boundary_bdev = bh->b_bdev;
530 }
531 bdev = bh->b_bdev;
532 } while ((bh = bh->b_this_page) != head);
533
534 if (first_unmapped)
535 goto page_is_mapped;
536
537
538
539
540
541
542
543 goto confused;
544 }
545
546
547
548
549 BUG_ON(!PageUptodate(page));
550 block_in_file = (sector_t)page->index << (PAGE_SHIFT - blkbits);
551 last_block = (i_size - 1) >> blkbits;
552 map_bh.b_page = page;
553 for (page_block = 0; page_block < blocks_per_page; ) {
554
555 map_bh.b_state = 0;
556 map_bh.b_size = 1 << blkbits;
557 if (mpd->get_block(inode, block_in_file, &map_bh, 1))
558 goto confused;
559 if (buffer_new(&map_bh))
560 clean_bdev_bh_alias(&map_bh);
561 if (buffer_boundary(&map_bh)) {
562 boundary_block = map_bh.b_blocknr;
563 boundary_bdev = map_bh.b_bdev;
564 }
565 if (page_block) {
566 if (map_bh.b_blocknr != blocks[page_block-1] + 1)
567 goto confused;
568 }
569 blocks[page_block++] = map_bh.b_blocknr;
570 boundary = buffer_boundary(&map_bh);
571 bdev = map_bh.b_bdev;
572 if (block_in_file == last_block)
573 break;
574 block_in_file++;
575 }
576 BUG_ON(page_block == 0);
577
578 first_unmapped = page_block;
579
580page_is_mapped:
581 end_index = i_size >> PAGE_SHIFT;
582 if (page->index >= end_index) {
583
584
585
586
587
588
589
590
591 unsigned offset = i_size & (PAGE_SIZE - 1);
592
593 if (page->index > end_index || !offset)
594 goto confused;
595 zero_user_segment(page, offset, PAGE_SIZE);
596 }
597
598
599
600
601 if (bio && mpd->last_block_in_bio != blocks[0] - 1)
602 bio = mpage_bio_submit(REQ_OP_WRITE, op_flags, bio);
603
604alloc_new:
605 if (bio == NULL) {
606 if (first_unmapped == blocks_per_page) {
607 if (!bdev_write_page(bdev, blocks[0] << (blkbits - 9),
608 page, wbc))
609 goto out;
610 }
611 bio = mpage_alloc(bdev, blocks[0] << (blkbits - 9),
612 BIO_MAX_VECS, GFP_NOFS|__GFP_HIGH);
613 if (bio == NULL)
614 goto confused;
615
616 wbc_init_bio(wbc, bio);
617 bio->bi_write_hint = inode->i_write_hint;
618 }
619
620
621
622
623
624
625 wbc_account_cgroup_owner(wbc, page, PAGE_SIZE);
626 length = first_unmapped << blkbits;
627 if (bio_add_page(bio, page, length, 0) < length) {
628 bio = mpage_bio_submit(REQ_OP_WRITE, op_flags, bio);
629 goto alloc_new;
630 }
631
632 clean_buffers(page, first_unmapped);
633
634 BUG_ON(PageWriteback(page));
635 set_page_writeback(page);
636 unlock_page(page);
637 if (boundary || (first_unmapped != blocks_per_page)) {
638 bio = mpage_bio_submit(REQ_OP_WRITE, op_flags, bio);
639 if (boundary_block) {
640 write_boundary_block(boundary_bdev,
641 boundary_block, 1 << blkbits);
642 }
643 } else {
644 mpd->last_block_in_bio = blocks[blocks_per_page - 1];
645 }
646 goto out;
647
648confused:
649 if (bio)
650 bio = mpage_bio_submit(REQ_OP_WRITE, op_flags, bio);
651
652 if (mpd->use_writepage) {
653 ret = mapping->a_ops->writepage(page, wbc);
654 } else {
655 ret = -EAGAIN;
656 goto out;
657 }
658
659
660
661 mapping_set_error(mapping, ret);
662out:
663 mpd->bio = bio;
664 return ret;
665}
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686int
687mpage_writepages(struct address_space *mapping,
688 struct writeback_control *wbc, get_block_t get_block)
689{
690 struct blk_plug plug;
691 int ret;
692
693 blk_start_plug(&plug);
694
695 if (!get_block)
696 ret = generic_writepages(mapping, wbc);
697 else {
698 struct mpage_data mpd = {
699 .bio = NULL,
700 .last_block_in_bio = 0,
701 .get_block = get_block,
702 .use_writepage = 1,
703 };
704
705 ret = write_cache_pages(mapping, wbc, __mpage_writepage, &mpd);
706 if (mpd.bio) {
707 int op_flags = (wbc->sync_mode == WB_SYNC_ALL ?
708 REQ_SYNC : 0);
709 mpage_bio_submit(REQ_OP_WRITE, op_flags, mpd.bio);
710 }
711 }
712 blk_finish_plug(&plug);
713 return ret;
714}
715EXPORT_SYMBOL(mpage_writepages);
716
717int mpage_writepage(struct page *page, get_block_t get_block,
718 struct writeback_control *wbc)
719{
720 struct mpage_data mpd = {
721 .bio = NULL,
722 .last_block_in_bio = 0,
723 .get_block = get_block,
724 .use_writepage = 0,
725 };
726 int ret = __mpage_writepage(page, wbc, &mpd);
727 if (mpd.bio) {
728 int op_flags = (wbc->sync_mode == WB_SYNC_ALL ?
729 REQ_SYNC : 0);
730 mpage_bio_submit(REQ_OP_WRITE, op_flags, mpd.bio);
731 }
732 return ret;
733}
734EXPORT_SYMBOL(mpage_writepage);
735