1
2
3
4
5
6
7
8
9
10
11
12
13
14
15#include <linux/kernel.h>
16#include <linux/module.h>
17#include <linux/mm.h>
18#include <linux/kdev_t.h>
19#include <linux/bio.h>
20#include <linux/fs.h>
21#include <linux/buffer_head.h>
22#include <linux/blkdev.h>
23#include <linux/highmem.h>
24#include <linux/prefetch.h>
25#include <linux/mpage.h>
26#include <linux/writeback.h>
27#include <linux/backing-dev.h>
28#include <linux/pagevec.h>
29
30
31
32
33
34
35
36
37
38
39
40
41
42static void mpage_end_io_read(struct bio *bio, int err)
43{
44 const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
45 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
46
47 do {
48 struct page *page = bvec->bv_page;
49
50 if (--bvec >= bio->bi_io_vec)
51 prefetchw(&bvec->bv_page->flags);
52
53 if (uptodate) {
54 SetPageUptodate(page);
55 } else {
56 ClearPageUptodate(page);
57 SetPageError(page);
58 }
59 unlock_page(page);
60 } while (bvec >= bio->bi_io_vec);
61 bio_put(bio);
62}
63
64static void mpage_end_io_write(struct bio *bio, int err)
65{
66 const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
67 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
68
69 do {
70 struct page *page = bvec->bv_page;
71
72 if (--bvec >= bio->bi_io_vec)
73 prefetchw(&bvec->bv_page->flags);
74
75 if (!uptodate){
76 SetPageError(page);
77 if (page->mapping)
78 set_bit(AS_EIO, &page->mapping->flags);
79 }
80 end_page_writeback(page);
81 } while (bvec >= bio->bi_io_vec);
82 bio_put(bio);
83}
84
85static struct bio *mpage_bio_submit(int rw, struct bio *bio)
86{
87 bio->bi_end_io = mpage_end_io_read;
88 if (rw == WRITE)
89 bio->bi_end_io = mpage_end_io_write;
90 submit_bio(rw, bio);
91 return NULL;
92}
93
94static struct bio *
95mpage_alloc(struct block_device *bdev,
96 sector_t first_sector, int nr_vecs,
97 gfp_t gfp_flags)
98{
99 struct bio *bio;
100
101 bio = bio_alloc(gfp_flags, nr_vecs);
102
103 if (bio == NULL && (current->flags & PF_MEMALLOC)) {
104 while (!bio && (nr_vecs /= 2))
105 bio = bio_alloc(gfp_flags, nr_vecs);
106 }
107
108 if (bio) {
109 bio->bi_bdev = bdev;
110 bio->bi_sector = first_sector;
111 }
112 return bio;
113}
114
115
116
117
118
119
120
121
122
123
124
125static void
126map_buffer_to_page(struct page *page, struct buffer_head *bh, int page_block)
127{
128 struct inode *inode = page->mapping->host;
129 struct buffer_head *page_bh, *head;
130 int block = 0;
131
132 if (!page_has_buffers(page)) {
133
134
135
136
137 if (inode->i_blkbits == PAGE_CACHE_SHIFT &&
138 buffer_uptodate(bh)) {
139 SetPageUptodate(page);
140 return;
141 }
142 create_empty_buffers(page, 1 << inode->i_blkbits, 0);
143 }
144 head = page_buffers(page);
145 page_bh = head;
146 do {
147 if (block == page_block) {
148 page_bh->b_state = bh->b_state;
149 page_bh->b_bdev = bh->b_bdev;
150 page_bh->b_blocknr = bh->b_blocknr;
151 break;
152 }
153 page_bh = page_bh->b_this_page;
154 block++;
155 } while (page_bh != head);
156}
157
158
159
160
161
162
163
164
165
166
167static struct bio *
168do_mpage_readpage(struct bio *bio, struct page *page, unsigned nr_pages,
169 sector_t *last_block_in_bio, struct buffer_head *map_bh,
170 unsigned long *first_logical_block, get_block_t get_block)
171{
172 struct inode *inode = page->mapping->host;
173 const unsigned blkbits = inode->i_blkbits;
174 const unsigned blocks_per_page = PAGE_CACHE_SIZE >> blkbits;
175 const unsigned blocksize = 1 << blkbits;
176 sector_t block_in_file;
177 sector_t last_block;
178 sector_t last_block_in_file;
179 sector_t blocks[MAX_BUF_PER_PAGE];
180 unsigned page_block;
181 unsigned first_hole = blocks_per_page;
182 struct block_device *bdev = NULL;
183 int length;
184 int fully_mapped = 1;
185 unsigned nblocks;
186 unsigned relative_block;
187
188 if (page_has_buffers(page))
189 goto confused;
190
191 block_in_file = (sector_t)page->index << (PAGE_CACHE_SHIFT - blkbits);
192 last_block = block_in_file + nr_pages * blocks_per_page;
193 last_block_in_file = (i_size_read(inode) + blocksize - 1) >> blkbits;
194 if (last_block > last_block_in_file)
195 last_block = last_block_in_file;
196 page_block = 0;
197
198
199
200
201 nblocks = map_bh->b_size >> blkbits;
202 if (buffer_mapped(map_bh) && block_in_file > *first_logical_block &&
203 block_in_file < (*first_logical_block + nblocks)) {
204 unsigned map_offset = block_in_file - *first_logical_block;
205 unsigned last = nblocks - map_offset;
206
207 for (relative_block = 0; ; relative_block++) {
208 if (relative_block == last) {
209 clear_buffer_mapped(map_bh);
210 break;
211 }
212 if (page_block == blocks_per_page)
213 break;
214 blocks[page_block] = map_bh->b_blocknr + map_offset +
215 relative_block;
216 page_block++;
217 block_in_file++;
218 }
219 bdev = map_bh->b_bdev;
220 }
221
222
223
224
225 map_bh->b_page = page;
226 while (page_block < blocks_per_page) {
227 map_bh->b_state = 0;
228 map_bh->b_size = 0;
229
230 if (block_in_file < last_block) {
231 map_bh->b_size = (last_block-block_in_file) << blkbits;
232 if (get_block(inode, block_in_file, map_bh, 0))
233 goto confused;
234 *first_logical_block = block_in_file;
235 }
236
237 if (!buffer_mapped(map_bh)) {
238 fully_mapped = 0;
239 if (first_hole == blocks_per_page)
240 first_hole = page_block;
241 page_block++;
242 block_in_file++;
243 continue;
244 }
245
246
247
248
249
250
251
252 if (buffer_uptodate(map_bh)) {
253 map_buffer_to_page(page, map_bh, page_block);
254 goto confused;
255 }
256
257 if (first_hole != blocks_per_page)
258 goto confused;
259
260
261 if (page_block && blocks[page_block-1] != map_bh->b_blocknr-1)
262 goto confused;
263 nblocks = map_bh->b_size >> blkbits;
264 for (relative_block = 0; ; relative_block++) {
265 if (relative_block == nblocks) {
266 clear_buffer_mapped(map_bh);
267 break;
268 } else if (page_block == blocks_per_page)
269 break;
270 blocks[page_block] = map_bh->b_blocknr+relative_block;
271 page_block++;
272 block_in_file++;
273 }
274 bdev = map_bh->b_bdev;
275 }
276
277 if (first_hole != blocks_per_page) {
278 zero_user_segment(page, first_hole << blkbits, PAGE_CACHE_SIZE);
279 if (first_hole == 0) {
280 SetPageUptodate(page);
281 unlock_page(page);
282 goto out;
283 }
284 } else if (fully_mapped) {
285 SetPageMappedToDisk(page);
286 }
287
288
289
290
291 if (bio && (*last_block_in_bio != blocks[0] - 1))
292 bio = mpage_bio_submit(READ, bio);
293
294alloc_new:
295 if (bio == NULL) {
296 bio = mpage_alloc(bdev, blocks[0] << (blkbits - 9),
297 min_t(int, nr_pages, bio_get_nr_vecs(bdev)),
298 GFP_KERNEL);
299 if (bio == NULL)
300 goto confused;
301 }
302
303 length = first_hole << blkbits;
304 if (bio_add_page(bio, page, length, 0) < length) {
305 bio = mpage_bio_submit(READ, bio);
306 goto alloc_new;
307 }
308
309 relative_block = block_in_file - *first_logical_block;
310 nblocks = map_bh->b_size >> blkbits;
311 if ((buffer_boundary(map_bh) && relative_block == nblocks) ||
312 (first_hole != blocks_per_page))
313 bio = mpage_bio_submit(READ, bio);
314 else
315 *last_block_in_bio = blocks[blocks_per_page - 1];
316out:
317 return bio;
318
319confused:
320 if (bio)
321 bio = mpage_bio_submit(READ, bio);
322 if (!PageUptodate(page))
323 block_read_full_page(page, get_block);
324 else
325 unlock_page(page);
326 goto out;
327}
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372int
373mpage_readpages(struct address_space *mapping, struct list_head *pages,
374 unsigned nr_pages, get_block_t get_block)
375{
376 struct bio *bio = NULL;
377 unsigned page_idx;
378 sector_t last_block_in_bio = 0;
379 struct buffer_head map_bh;
380 unsigned long first_logical_block = 0;
381
382 map_bh.b_state = 0;
383 map_bh.b_size = 0;
384 for (page_idx = 0; page_idx < nr_pages; page_idx++) {
385 struct page *page = list_entry(pages->prev, struct page, lru);
386
387 prefetchw(&page->flags);
388 list_del(&page->lru);
389 if (!add_to_page_cache_lru(page, mapping,
390 page->index, GFP_KERNEL)) {
391 bio = do_mpage_readpage(bio, page,
392 nr_pages - page_idx,
393 &last_block_in_bio, &map_bh,
394 &first_logical_block,
395 get_block);
396 }
397 page_cache_release(page);
398 }
399 BUG_ON(!list_empty(pages));
400 if (bio)
401 mpage_bio_submit(READ, bio);
402 return 0;
403}
404EXPORT_SYMBOL(mpage_readpages);
405
406
407
408
409int mpage_readpage(struct page *page, get_block_t get_block)
410{
411 struct bio *bio = NULL;
412 sector_t last_block_in_bio = 0;
413 struct buffer_head map_bh;
414 unsigned long first_logical_block = 0;
415
416 map_bh.b_state = 0;
417 map_bh.b_size = 0;
418 bio = do_mpage_readpage(bio, page, 1, &last_block_in_bio,
419 &map_bh, &first_logical_block, get_block);
420 if (bio)
421 mpage_bio_submit(READ, bio);
422 return 0;
423}
424EXPORT_SYMBOL(mpage_readpage);
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443struct mpage_data {
444 struct bio *bio;
445 sector_t last_block_in_bio;
446 get_block_t *get_block;
447 unsigned use_writepage;
448};
449
450static int __mpage_writepage(struct page *page, struct writeback_control *wbc,
451 void *data)
452{
453 struct mpage_data *mpd = data;
454 struct bio *bio = mpd->bio;
455 struct address_space *mapping = page->mapping;
456 struct inode *inode = page->mapping->host;
457 const unsigned blkbits = inode->i_blkbits;
458 unsigned long end_index;
459 const unsigned blocks_per_page = PAGE_CACHE_SIZE >> blkbits;
460 sector_t last_block;
461 sector_t block_in_file;
462 sector_t blocks[MAX_BUF_PER_PAGE];
463 unsigned page_block;
464 unsigned first_unmapped = blocks_per_page;
465 struct block_device *bdev = NULL;
466 int boundary = 0;
467 sector_t boundary_block = 0;
468 struct block_device *boundary_bdev = NULL;
469 int length;
470 struct buffer_head map_bh;
471 loff_t i_size = i_size_read(inode);
472 int ret = 0;
473
474 if (page_has_buffers(page)) {
475 struct buffer_head *head = page_buffers(page);
476 struct buffer_head *bh = head;
477
478
479 page_block = 0;
480 do {
481 BUG_ON(buffer_locked(bh));
482 if (!buffer_mapped(bh)) {
483
484
485
486
487 if (buffer_dirty(bh))
488 goto confused;
489 if (first_unmapped == blocks_per_page)
490 first_unmapped = page_block;
491 continue;
492 }
493
494 if (first_unmapped != blocks_per_page)
495 goto confused;
496
497 if (!buffer_dirty(bh) || !buffer_uptodate(bh))
498 goto confused;
499 if (page_block) {
500 if (bh->b_blocknr != blocks[page_block-1] + 1)
501 goto confused;
502 }
503 blocks[page_block++] = bh->b_blocknr;
504 boundary = buffer_boundary(bh);
505 if (boundary) {
506 boundary_block = bh->b_blocknr;
507 boundary_bdev = bh->b_bdev;
508 }
509 bdev = bh->b_bdev;
510 } while ((bh = bh->b_this_page) != head);
511
512 if (first_unmapped)
513 goto page_is_mapped;
514
515
516
517
518
519
520
521 goto confused;
522 }
523
524
525
526
527 BUG_ON(!PageUptodate(page));
528 block_in_file = (sector_t)page->index << (PAGE_CACHE_SHIFT - blkbits);
529 last_block = (i_size - 1) >> blkbits;
530 map_bh.b_page = page;
531 for (page_block = 0; page_block < blocks_per_page; ) {
532
533 map_bh.b_state = 0;
534 map_bh.b_size = 1 << blkbits;
535 if (mpd->get_block(inode, block_in_file, &map_bh, 1))
536 goto confused;
537 if (buffer_new(&map_bh))
538 unmap_underlying_metadata(map_bh.b_bdev,
539 map_bh.b_blocknr);
540 if (buffer_boundary(&map_bh)) {
541 boundary_block = map_bh.b_blocknr;
542 boundary_bdev = map_bh.b_bdev;
543 }
544 if (page_block) {
545 if (map_bh.b_blocknr != blocks[page_block-1] + 1)
546 goto confused;
547 }
548 blocks[page_block++] = map_bh.b_blocknr;
549 boundary = buffer_boundary(&map_bh);
550 bdev = map_bh.b_bdev;
551 if (block_in_file == last_block)
552 break;
553 block_in_file++;
554 }
555 BUG_ON(page_block == 0);
556
557 first_unmapped = page_block;
558
559page_is_mapped:
560 end_index = i_size >> PAGE_CACHE_SHIFT;
561 if (page->index >= end_index) {
562
563
564
565
566
567
568
569
570 unsigned offset = i_size & (PAGE_CACHE_SIZE - 1);
571
572 if (page->index > end_index || !offset)
573 goto confused;
574 zero_user_segment(page, offset, PAGE_CACHE_SIZE);
575 }
576
577
578
579
580 if (bio && mpd->last_block_in_bio != blocks[0] - 1)
581 bio = mpage_bio_submit(WRITE, bio);
582
583alloc_new:
584 if (bio == NULL) {
585 bio = mpage_alloc(bdev, blocks[0] << (blkbits - 9),
586 bio_get_nr_vecs(bdev), GFP_NOFS|__GFP_HIGH);
587 if (bio == NULL)
588 goto confused;
589 }
590
591
592
593
594
595
596 length = first_unmapped << blkbits;
597 if (bio_add_page(bio, page, length, 0) < length) {
598 bio = mpage_bio_submit(WRITE, bio);
599 goto alloc_new;
600 }
601
602
603
604
605
606 if (page_has_buffers(page)) {
607 struct buffer_head *head = page_buffers(page);
608 struct buffer_head *bh = head;
609 unsigned buffer_counter = 0;
610
611 do {
612 if (buffer_counter++ == first_unmapped)
613 break;
614 clear_buffer_dirty(bh);
615 bh = bh->b_this_page;
616 } while (bh != head);
617
618
619
620
621
622
623 if (buffer_heads_over_limit && PageUptodate(page))
624 try_to_free_buffers(page);
625 }
626
627 BUG_ON(PageWriteback(page));
628 set_page_writeback(page);
629 unlock_page(page);
630 if (boundary || (first_unmapped != blocks_per_page)) {
631 bio = mpage_bio_submit(WRITE, bio);
632 if (boundary_block) {
633 write_boundary_block(boundary_bdev,
634 boundary_block, 1 << blkbits);
635 }
636 } else {
637 mpd->last_block_in_bio = blocks[blocks_per_page - 1];
638 }
639 goto out;
640
641confused:
642 if (bio)
643 bio = mpage_bio_submit(WRITE, bio);
644
645 if (mpd->use_writepage) {
646 ret = mapping->a_ops->writepage(page, wbc);
647 } else {
648 ret = -EAGAIN;
649 goto out;
650 }
651
652
653
654 mapping_set_error(mapping, ret);
655out:
656 mpd->bio = bio;
657 return ret;
658}
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679int
680mpage_writepages(struct address_space *mapping,
681 struct writeback_control *wbc, get_block_t get_block)
682{
683 int ret;
684
685 if (!get_block)
686 ret = generic_writepages(mapping, wbc);
687 else {
688 struct mpage_data mpd = {
689 .bio = NULL,
690 .last_block_in_bio = 0,
691 .get_block = get_block,
692 .use_writepage = 1,
693 };
694
695 ret = write_cache_pages(mapping, wbc, __mpage_writepage, &mpd);
696 if (mpd.bio)
697 mpage_bio_submit(WRITE, mpd.bio);
698 }
699 return ret;
700}
701EXPORT_SYMBOL(mpage_writepages);
702
703int mpage_writepage(struct page *page, get_block_t get_block,
704 struct writeback_control *wbc)
705{
706 struct mpage_data mpd = {
707 .bio = NULL,
708 .last_block_in_bio = 0,
709 .get_block = get_block,
710 .use_writepage = 0,
711 };
712 int ret = __mpage_writepage(page, wbc, &mpd);
713 if (mpd.bio)
714 mpage_bio_submit(WRITE, mpd.bio);
715 return ret;
716}
717EXPORT_SYMBOL(mpage_writepage);
718