1
2
3
4
5
6
7
8
9
10
11
12
13
14
15#include <linux/kernel.h>
16#include <linux/export.h>
17#include <linux/mm.h>
18#include <linux/kdev_t.h>
19#include <linux/gfp.h>
20#include <linux/bio.h>
21#include <linux/fs.h>
22#include <linux/buffer_head.h>
23#include <linux/blkdev.h>
24#include <linux/highmem.h>
25#include <linux/prefetch.h>
26#include <linux/mpage.h>
27#include <linux/writeback.h>
28#include <linux/backing-dev.h>
29#include <linux/pagevec.h>
30#include <linux/cleancache.h>
31#include "internal.h"
32
33
34
35
36
37
38
39
40
41
42
43
44
45static void mpage_end_io(struct bio *bio)
46{
47 struct bio_vec *bv;
48 int i;
49
50 bio_for_each_segment_all(bv, bio, i) {
51 struct page *page = bv->bv_page;
52 page_endio(page, bio_data_dir(bio), bio->bi_error);
53 }
54
55 bio_put(bio);
56}
57
58static struct bio *mpage_bio_submit(int rw, struct bio *bio)
59{
60 bio->bi_end_io = mpage_end_io;
61 guard_bio_eod(rw, bio);
62 submit_bio(rw, bio);
63 return NULL;
64}
65
66static struct bio *
67mpage_alloc(struct block_device *bdev,
68 sector_t first_sector, int nr_vecs,
69 gfp_t gfp_flags)
70{
71 struct bio *bio;
72
73 bio = bio_alloc(gfp_flags, nr_vecs);
74
75 if (bio == NULL && (current->flags & PF_MEMALLOC)) {
76 while (!bio && (nr_vecs /= 2))
77 bio = bio_alloc(gfp_flags, nr_vecs);
78 }
79
80 if (bio) {
81 bio->bi_bdev = bdev;
82 bio->bi_iter.bi_sector = first_sector;
83 }
84 return bio;
85}
86
87
88
89
90
91
92
93
94
95
96
97static void
98map_buffer_to_page(struct page *page, struct buffer_head *bh, int page_block)
99{
100 struct inode *inode = page->mapping->host;
101 struct buffer_head *page_bh, *head;
102 int block = 0;
103
104 if (!page_has_buffers(page)) {
105
106
107
108
109 if (inode->i_blkbits == PAGE_CACHE_SHIFT &&
110 buffer_uptodate(bh)) {
111 SetPageUptodate(page);
112 return;
113 }
114 create_empty_buffers(page, 1 << inode->i_blkbits, 0);
115 }
116 head = page_buffers(page);
117 page_bh = head;
118 do {
119 if (block == page_block) {
120 page_bh->b_state = bh->b_state;
121 page_bh->b_bdev = bh->b_bdev;
122 page_bh->b_blocknr = bh->b_blocknr;
123 break;
124 }
125 page_bh = page_bh->b_this_page;
126 block++;
127 } while (page_bh != head);
128}
129
130
131
132
133
134
135
136
137
138
139static struct bio *
140do_mpage_readpage(struct bio *bio, struct page *page, unsigned nr_pages,
141 sector_t *last_block_in_bio, struct buffer_head *map_bh,
142 unsigned long *first_logical_block, get_block_t get_block,
143 gfp_t gfp)
144{
145 struct inode *inode = page->mapping->host;
146 const unsigned blkbits = inode->i_blkbits;
147 const unsigned blocks_per_page = PAGE_CACHE_SIZE >> blkbits;
148 const unsigned blocksize = 1 << blkbits;
149 sector_t block_in_file;
150 sector_t last_block;
151 sector_t last_block_in_file;
152 sector_t blocks[MAX_BUF_PER_PAGE];
153 unsigned page_block;
154 unsigned first_hole = blocks_per_page;
155 struct block_device *bdev = NULL;
156 int length;
157 int fully_mapped = 1;
158 unsigned nblocks;
159 unsigned relative_block;
160
161 if (page_has_buffers(page))
162 goto confused;
163
164 block_in_file = (sector_t)page->index << (PAGE_CACHE_SHIFT - blkbits);
165 last_block = block_in_file + nr_pages * blocks_per_page;
166 last_block_in_file = (i_size_read(inode) + blocksize - 1) >> blkbits;
167 if (last_block > last_block_in_file)
168 last_block = last_block_in_file;
169 page_block = 0;
170
171
172
173
174 nblocks = map_bh->b_size >> blkbits;
175 if (buffer_mapped(map_bh) && block_in_file > *first_logical_block &&
176 block_in_file < (*first_logical_block + nblocks)) {
177 unsigned map_offset = block_in_file - *first_logical_block;
178 unsigned last = nblocks - map_offset;
179
180 for (relative_block = 0; ; relative_block++) {
181 if (relative_block == last) {
182 clear_buffer_mapped(map_bh);
183 break;
184 }
185 if (page_block == blocks_per_page)
186 break;
187 blocks[page_block] = map_bh->b_blocknr + map_offset +
188 relative_block;
189 page_block++;
190 block_in_file++;
191 }
192 bdev = map_bh->b_bdev;
193 }
194
195
196
197
198 map_bh->b_page = page;
199 while (page_block < blocks_per_page) {
200 map_bh->b_state = 0;
201 map_bh->b_size = 0;
202
203 if (block_in_file < last_block) {
204 map_bh->b_size = (last_block-block_in_file) << blkbits;
205 if (get_block(inode, block_in_file, map_bh, 0))
206 goto confused;
207 *first_logical_block = block_in_file;
208 }
209
210 if (!buffer_mapped(map_bh)) {
211 fully_mapped = 0;
212 if (first_hole == blocks_per_page)
213 first_hole = page_block;
214 page_block++;
215 block_in_file++;
216 continue;
217 }
218
219
220
221
222
223
224
225 if (buffer_uptodate(map_bh)) {
226 map_buffer_to_page(page, map_bh, page_block);
227 goto confused;
228 }
229
230 if (first_hole != blocks_per_page)
231 goto confused;
232
233
234 if (page_block && blocks[page_block-1] != map_bh->b_blocknr-1)
235 goto confused;
236 nblocks = map_bh->b_size >> blkbits;
237 for (relative_block = 0; ; relative_block++) {
238 if (relative_block == nblocks) {
239 clear_buffer_mapped(map_bh);
240 break;
241 } else if (page_block == blocks_per_page)
242 break;
243 blocks[page_block] = map_bh->b_blocknr+relative_block;
244 page_block++;
245 block_in_file++;
246 }
247 bdev = map_bh->b_bdev;
248 }
249
250 if (first_hole != blocks_per_page) {
251 zero_user_segment(page, first_hole << blkbits, PAGE_CACHE_SIZE);
252 if (first_hole == 0) {
253 SetPageUptodate(page);
254 unlock_page(page);
255 goto out;
256 }
257 } else if (fully_mapped) {
258 SetPageMappedToDisk(page);
259 }
260
261 if (fully_mapped && blocks_per_page == 1 && !PageUptodate(page) &&
262 cleancache_get_page(page) == 0) {
263 SetPageUptodate(page);
264 goto confused;
265 }
266
267
268
269
270 if (bio && (*last_block_in_bio != blocks[0] - 1))
271 bio = mpage_bio_submit(READ, bio);
272
273alloc_new:
274 if (bio == NULL) {
275 if (first_hole == blocks_per_page) {
276 if (!bdev_read_page(bdev, blocks[0] << (blkbits - 9),
277 page))
278 goto out;
279 }
280 bio = mpage_alloc(bdev, blocks[0] << (blkbits - 9),
281 min_t(int, nr_pages, BIO_MAX_PAGES), gfp);
282 if (bio == NULL)
283 goto confused;
284 }
285
286 length = first_hole << blkbits;
287 if (bio_add_page(bio, page, length, 0) < length) {
288 bio = mpage_bio_submit(READ, bio);
289 goto alloc_new;
290 }
291
292 relative_block = block_in_file - *first_logical_block;
293 nblocks = map_bh->b_size >> blkbits;
294 if ((buffer_boundary(map_bh) && relative_block == nblocks) ||
295 (first_hole != blocks_per_page))
296 bio = mpage_bio_submit(READ, bio);
297 else
298 *last_block_in_bio = blocks[blocks_per_page - 1];
299out:
300 return bio;
301
302confused:
303 if (bio)
304 bio = mpage_bio_submit(READ, bio);
305 if (!PageUptodate(page))
306 block_read_full_page(page, get_block);
307 else
308 unlock_page(page);
309 goto out;
310}
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355int
356mpage_readpages(struct address_space *mapping, struct list_head *pages,
357 unsigned nr_pages, get_block_t get_block)
358{
359 struct bio *bio = NULL;
360 unsigned page_idx;
361 sector_t last_block_in_bio = 0;
362 struct buffer_head map_bh;
363 unsigned long first_logical_block = 0;
364 gfp_t gfp = mapping_gfp_constraint(mapping, GFP_KERNEL);
365
366 map_bh.b_state = 0;
367 map_bh.b_size = 0;
368 for (page_idx = 0; page_idx < nr_pages; page_idx++) {
369 struct page *page = list_entry(pages->prev, struct page, lru);
370
371 prefetchw(&page->flags);
372 list_del(&page->lru);
373 if (!add_to_page_cache_lru(page, mapping,
374 page->index,
375 gfp)) {
376 bio = do_mpage_readpage(bio, page,
377 nr_pages - page_idx,
378 &last_block_in_bio, &map_bh,
379 &first_logical_block,
380 get_block, gfp);
381 }
382 page_cache_release(page);
383 }
384 BUG_ON(!list_empty(pages));
385 if (bio)
386 mpage_bio_submit(READ, bio);
387 return 0;
388}
389EXPORT_SYMBOL(mpage_readpages);
390
391
392
393
394int mpage_readpage(struct page *page, get_block_t get_block)
395{
396 struct bio *bio = NULL;
397 sector_t last_block_in_bio = 0;
398 struct buffer_head map_bh;
399 unsigned long first_logical_block = 0;
400 gfp_t gfp = mapping_gfp_constraint(page->mapping, GFP_KERNEL);
401
402 map_bh.b_state = 0;
403 map_bh.b_size = 0;
404 bio = do_mpage_readpage(bio, page, 1, &last_block_in_bio,
405 &map_bh, &first_logical_block, get_block, gfp);
406 if (bio)
407 mpage_bio_submit(READ, bio);
408 return 0;
409}
410EXPORT_SYMBOL(mpage_readpage);
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429struct mpage_data {
430 struct bio *bio;
431 sector_t last_block_in_bio;
432 get_block_t *get_block;
433 unsigned use_writepage;
434};
435
436
437
438
439
440static void clean_buffers(struct page *page, unsigned first_unmapped)
441{
442 unsigned buffer_counter = 0;
443 struct buffer_head *bh, *head;
444 if (!page_has_buffers(page))
445 return;
446 head = page_buffers(page);
447 bh = head;
448
449 do {
450 if (buffer_counter++ == first_unmapped)
451 break;
452 clear_buffer_dirty(bh);
453 bh = bh->b_this_page;
454 } while (bh != head);
455
456
457
458
459
460
461 if (buffer_heads_over_limit && PageUptodate(page))
462 try_to_free_buffers(page);
463}
464
465static int __mpage_writepage(struct page *page, struct writeback_control *wbc,
466 void *data)
467{
468 struct mpage_data *mpd = data;
469 struct bio *bio = mpd->bio;
470 struct address_space *mapping = page->mapping;
471 struct inode *inode = page->mapping->host;
472 const unsigned blkbits = inode->i_blkbits;
473 unsigned long end_index;
474 const unsigned blocks_per_page = PAGE_CACHE_SIZE >> blkbits;
475 sector_t last_block;
476 sector_t block_in_file;
477 sector_t blocks[MAX_BUF_PER_PAGE];
478 unsigned page_block;
479 unsigned first_unmapped = blocks_per_page;
480 struct block_device *bdev = NULL;
481 int boundary = 0;
482 sector_t boundary_block = 0;
483 struct block_device *boundary_bdev = NULL;
484 int length;
485 struct buffer_head map_bh;
486 loff_t i_size = i_size_read(inode);
487 int ret = 0;
488 int wr = (wbc->sync_mode == WB_SYNC_ALL ? WRITE_SYNC : WRITE);
489
490 if (page_has_buffers(page)) {
491 struct buffer_head *head = page_buffers(page);
492 struct buffer_head *bh = head;
493
494
495 page_block = 0;
496 do {
497 BUG_ON(buffer_locked(bh));
498 if (!buffer_mapped(bh)) {
499
500
501
502
503 if (buffer_dirty(bh))
504 goto confused;
505 if (first_unmapped == blocks_per_page)
506 first_unmapped = page_block;
507 continue;
508 }
509
510 if (first_unmapped != blocks_per_page)
511 goto confused;
512
513 if (!buffer_dirty(bh) || !buffer_uptodate(bh))
514 goto confused;
515 if (page_block) {
516 if (bh->b_blocknr != blocks[page_block-1] + 1)
517 goto confused;
518 }
519 blocks[page_block++] = bh->b_blocknr;
520 boundary = buffer_boundary(bh);
521 if (boundary) {
522 boundary_block = bh->b_blocknr;
523 boundary_bdev = bh->b_bdev;
524 }
525 bdev = bh->b_bdev;
526 } while ((bh = bh->b_this_page) != head);
527
528 if (first_unmapped)
529 goto page_is_mapped;
530
531
532
533
534
535
536
537 goto confused;
538 }
539
540
541
542
543 BUG_ON(!PageUptodate(page));
544 block_in_file = (sector_t)page->index << (PAGE_CACHE_SHIFT - blkbits);
545 last_block = (i_size - 1) >> blkbits;
546 map_bh.b_page = page;
547 for (page_block = 0; page_block < blocks_per_page; ) {
548
549 map_bh.b_state = 0;
550 map_bh.b_size = 1 << blkbits;
551 if (mpd->get_block(inode, block_in_file, &map_bh, 1))
552 goto confused;
553 if (buffer_new(&map_bh))
554 unmap_underlying_metadata(map_bh.b_bdev,
555 map_bh.b_blocknr);
556 if (buffer_boundary(&map_bh)) {
557 boundary_block = map_bh.b_blocknr;
558 boundary_bdev = map_bh.b_bdev;
559 }
560 if (page_block) {
561 if (map_bh.b_blocknr != blocks[page_block-1] + 1)
562 goto confused;
563 }
564 blocks[page_block++] = map_bh.b_blocknr;
565 boundary = buffer_boundary(&map_bh);
566 bdev = map_bh.b_bdev;
567 if (block_in_file == last_block)
568 break;
569 block_in_file++;
570 }
571 BUG_ON(page_block == 0);
572
573 first_unmapped = page_block;
574
575page_is_mapped:
576 end_index = i_size >> PAGE_CACHE_SHIFT;
577 if (page->index >= end_index) {
578
579
580
581
582
583
584
585
586 unsigned offset = i_size & (PAGE_CACHE_SIZE - 1);
587
588 if (page->index > end_index || !offset)
589 goto confused;
590 zero_user_segment(page, offset, PAGE_CACHE_SIZE);
591 }
592
593
594
595
596 if (bio && mpd->last_block_in_bio != blocks[0] - 1)
597 bio = mpage_bio_submit(wr, bio);
598
599alloc_new:
600 if (bio == NULL) {
601 if (first_unmapped == blocks_per_page) {
602 if (!bdev_write_page(bdev, blocks[0] << (blkbits - 9),
603 page, wbc)) {
604 clean_buffers(page, first_unmapped);
605 goto out;
606 }
607 }
608 bio = mpage_alloc(bdev, blocks[0] << (blkbits - 9),
609 BIO_MAX_PAGES, GFP_NOFS|__GFP_HIGH);
610 if (bio == NULL)
611 goto confused;
612
613 wbc_init_bio(wbc, bio);
614 }
615
616
617
618
619
620
621 wbc_account_io(wbc, page, PAGE_SIZE);
622 length = first_unmapped << blkbits;
623 if (bio_add_page(bio, page, length, 0) < length) {
624 bio = mpage_bio_submit(wr, bio);
625 goto alloc_new;
626 }
627
628 clean_buffers(page, first_unmapped);
629
630 BUG_ON(PageWriteback(page));
631 set_page_writeback(page);
632 unlock_page(page);
633 if (boundary || (first_unmapped != blocks_per_page)) {
634 bio = mpage_bio_submit(wr, bio);
635 if (boundary_block) {
636 write_boundary_block(boundary_bdev,
637 boundary_block, 1 << blkbits);
638 }
639 } else {
640 mpd->last_block_in_bio = blocks[blocks_per_page - 1];
641 }
642 goto out;
643
644confused:
645 if (bio)
646 bio = mpage_bio_submit(wr, bio);
647
648 if (mpd->use_writepage) {
649 ret = mapping->a_ops->writepage(page, wbc);
650 } else {
651 ret = -EAGAIN;
652 goto out;
653 }
654
655
656
657 mapping_set_error(mapping, ret);
658out:
659 mpd->bio = bio;
660 return ret;
661}
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682int
683mpage_writepages(struct address_space *mapping,
684 struct writeback_control *wbc, get_block_t get_block)
685{
686 struct blk_plug plug;
687 int ret;
688
689 blk_start_plug(&plug);
690
691 if (!get_block)
692 ret = generic_writepages(mapping, wbc);
693 else {
694 struct mpage_data mpd = {
695 .bio = NULL,
696 .last_block_in_bio = 0,
697 .get_block = get_block,
698 .use_writepage = 1,
699 };
700
701 ret = write_cache_pages(mapping, wbc, __mpage_writepage, &mpd);
702 if (mpd.bio) {
703 int wr = (wbc->sync_mode == WB_SYNC_ALL ?
704 WRITE_SYNC : WRITE);
705 mpage_bio_submit(wr, mpd.bio);
706 }
707 }
708 blk_finish_plug(&plug);
709 return ret;
710}
711EXPORT_SYMBOL(mpage_writepages);
712
713int mpage_writepage(struct page *page, get_block_t get_block,
714 struct writeback_control *wbc)
715{
716 struct mpage_data mpd = {
717 .bio = NULL,
718 .last_block_in_bio = 0,
719 .get_block = get_block,
720 .use_writepage = 0,
721 };
722 int ret = __mpage_writepage(page, wbc, &mpd);
723 if (mpd.bio) {
724 int wr = (wbc->sync_mode == WB_SYNC_ALL ?
725 WRITE_SYNC : WRITE);
726 mpage_bio_submit(wr, mpd.bio);
727 }
728 return ret;
729}
730EXPORT_SYMBOL(mpage_writepage);
731