1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18#include "xfs.h"
19#include <linux/stddef.h>
20#include <linux/errno.h>
21#include <linux/gfp.h>
22#include <linux/pagemap.h>
23#include <linux/init.h>
24#include <linux/vmalloc.h>
25#include <linux/bio.h>
26#include <linux/sysctl.h>
27#include <linux/proc_fs.h>
28#include <linux/workqueue.h>
29#include <linux/percpu.h>
30#include <linux/blkdev.h>
31#include <linux/hash.h>
32#include <linux/kthread.h>
33#include <linux/migrate.h>
34#include <linux/backing-dev.h>
35#include <linux/freezer.h>
36#include <linux/list_sort.h>
37
38#include "xfs_sb.h"
39#include "xfs_inum.h"
40#include "xfs_log.h"
41#include "xfs_ag.h"
42#include "xfs_mount.h"
43#include "xfs_trace.h"
44
45static kmem_zone_t *xfs_buf_zone;
46STATIC int xfsbufd(void *);
47STATIC void xfs_buf_delwri_queue(xfs_buf_t *, int);
48
49static struct workqueue_struct *xfslogd_workqueue;
50struct workqueue_struct *xfsdatad_workqueue;
51struct workqueue_struct *xfsconvertd_workqueue;
52
53#ifdef XFS_BUF_LOCK_TRACKING
54# define XB_SET_OWNER(bp) ((bp)->b_last_holder = current->pid)
55# define XB_CLEAR_OWNER(bp) ((bp)->b_last_holder = -1)
56# define XB_GET_OWNER(bp) ((bp)->b_last_holder)
57#else
58# define XB_SET_OWNER(bp) do { } while (0)
59# define XB_CLEAR_OWNER(bp) do { } while (0)
60# define XB_GET_OWNER(bp) do { } while (0)
61#endif
62
63#define xb_to_gfp(flags) \
64 ((((flags) & XBF_READ_AHEAD) ? __GFP_NORETRY : \
65 ((flags) & XBF_DONT_BLOCK) ? GFP_NOFS : GFP_KERNEL) | __GFP_NOWARN)
66
67#define xb_to_km(flags) \
68 (((flags) & XBF_DONT_BLOCK) ? KM_NOFS : KM_SLEEP)
69
70#define xfs_buf_allocate(flags) \
71 kmem_zone_alloc(xfs_buf_zone, xb_to_km(flags))
72#define xfs_buf_deallocate(bp) \
73 kmem_zone_free(xfs_buf_zone, (bp));
74
75static inline int
76xfs_buf_is_vmapped(
77 struct xfs_buf *bp)
78{
79
80
81
82
83
84
85
86 return (bp->b_flags & XBF_MAPPED) && bp->b_page_count > 1;
87}
88
89static inline int
90xfs_buf_vmap_len(
91 struct xfs_buf *bp)
92{
93 return (bp->b_page_count * PAGE_SIZE) - bp->b_offset;
94}
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109#if (BITS_PER_LONG == 32)
110#define PRSHIFT (PAGE_CACHE_SHIFT - 5)
111#elif (BITS_PER_LONG == 64)
112#define PRSHIFT (PAGE_CACHE_SHIFT - 6)
113#else
114#error BITS_PER_LONG must be 32 or 64
115#endif
116#define NBPPR (PAGE_CACHE_SIZE/BITS_PER_LONG)
117#define BTOPR(b) (((unsigned int)(b) + (NBPPR - 1)) >> PRSHIFT)
118#define BTOPRT(b) (((unsigned int)(b) >> PRSHIFT))
119
120STATIC unsigned long
121page_region_mask(
122 size_t offset,
123 size_t length)
124{
125 unsigned long mask;
126 int first, final;
127
128 first = BTOPR(offset);
129 final = BTOPRT(offset + length - 1);
130 first = min(first, final);
131
132 mask = ~0UL;
133 mask <<= BITS_PER_LONG - (final - first);
134 mask >>= BITS_PER_LONG - (final);
135
136 ASSERT(offset + length <= PAGE_CACHE_SIZE);
137 ASSERT((final - first) < BITS_PER_LONG && (final - first) >= 0);
138
139 return mask;
140}
141
142STATIC void
143set_page_region(
144 struct page *page,
145 size_t offset,
146 size_t length)
147{
148 set_page_private(page,
149 page_private(page) | page_region_mask(offset, length));
150 if (page_private(page) == ~0UL)
151 SetPageUptodate(page);
152}
153
154STATIC int
155test_page_region(
156 struct page *page,
157 size_t offset,
158 size_t length)
159{
160 unsigned long mask = page_region_mask(offset, length);
161
162 return (mask && (page_private(page) & mask) == mask);
163}
164
165
166
167
168
169
170
171STATIC void
172xfs_buf_lru_add(
173 struct xfs_buf *bp)
174{
175 struct xfs_buftarg *btp = bp->b_target;
176
177 spin_lock(&btp->bt_lru_lock);
178 if (list_empty(&bp->b_lru)) {
179 atomic_inc(&bp->b_hold);
180 list_add_tail(&bp->b_lru, &btp->bt_lru);
181 btp->bt_lru_nr++;
182 }
183 spin_unlock(&btp->bt_lru_lock);
184}
185
186
187
188
189
190
191
192
193
194
195STATIC void
196xfs_buf_lru_del(
197 struct xfs_buf *bp)
198{
199 struct xfs_buftarg *btp = bp->b_target;
200
201 if (list_empty(&bp->b_lru))
202 return;
203
204 spin_lock(&btp->bt_lru_lock);
205 if (!list_empty(&bp->b_lru)) {
206 list_del_init(&bp->b_lru);
207 btp->bt_lru_nr--;
208 }
209 spin_unlock(&btp->bt_lru_lock);
210}
211
212
213
214
215
216
217
218
219
220void
221xfs_buf_stale(
222 struct xfs_buf *bp)
223{
224 bp->b_flags |= XBF_STALE;
225 atomic_set(&(bp)->b_lru_ref, 0);
226 if (!list_empty(&bp->b_lru)) {
227 struct xfs_buftarg *btp = bp->b_target;
228
229 spin_lock(&btp->bt_lru_lock);
230 if (!list_empty(&bp->b_lru)) {
231 list_del_init(&bp->b_lru);
232 btp->bt_lru_nr--;
233 atomic_dec(&bp->b_hold);
234 }
235 spin_unlock(&btp->bt_lru_lock);
236 }
237 ASSERT(atomic_read(&bp->b_hold) >= 1);
238}
239
240STATIC void
241_xfs_buf_initialize(
242 xfs_buf_t *bp,
243 xfs_buftarg_t *target,
244 xfs_off_t range_base,
245 size_t range_length,
246 xfs_buf_flags_t flags)
247{
248
249
250
251 flags &= ~(XBF_LOCK|XBF_MAPPED|XBF_DONT_BLOCK|XBF_READ_AHEAD);
252
253 memset(bp, 0, sizeof(xfs_buf_t));
254 atomic_set(&bp->b_hold, 1);
255 atomic_set(&bp->b_lru_ref, 1);
256 init_completion(&bp->b_iowait);
257 INIT_LIST_HEAD(&bp->b_lru);
258 INIT_LIST_HEAD(&bp->b_list);
259 RB_CLEAR_NODE(&bp->b_rbnode);
260 sema_init(&bp->b_sema, 0);
261 XB_SET_OWNER(bp);
262 bp->b_target = target;
263 bp->b_file_offset = range_base;
264
265
266
267
268
269 bp->b_buffer_length = bp->b_count_desired = range_length;
270 bp->b_flags = flags;
271 bp->b_bn = XFS_BUF_DADDR_NULL;
272 atomic_set(&bp->b_pin_count, 0);
273 init_waitqueue_head(&bp->b_waiters);
274
275 XFS_STATS_INC(xb_create);
276
277 trace_xfs_buf_init(bp, _RET_IP_);
278}
279
280
281
282
283
284STATIC int
285_xfs_buf_get_pages(
286 xfs_buf_t *bp,
287 int page_count,
288 xfs_buf_flags_t flags)
289{
290
291 if (bp->b_pages == NULL) {
292 bp->b_offset = xfs_buf_poff(bp->b_file_offset);
293 bp->b_page_count = page_count;
294 if (page_count <= XB_PAGES) {
295 bp->b_pages = bp->b_page_array;
296 } else {
297 bp->b_pages = kmem_alloc(sizeof(struct page *) *
298 page_count, xb_to_km(flags));
299 if (bp->b_pages == NULL)
300 return -ENOMEM;
301 }
302 memset(bp->b_pages, 0, sizeof(struct page *) * page_count);
303 }
304 return 0;
305}
306
307
308
309
310STATIC void
311_xfs_buf_free_pages(
312 xfs_buf_t *bp)
313{
314 if (bp->b_pages != bp->b_page_array) {
315 kmem_free(bp->b_pages);
316 bp->b_pages = NULL;
317 }
318}
319
320
321
322
323
324
325
326
327void
328xfs_buf_free(
329 xfs_buf_t *bp)
330{
331 trace_xfs_buf_free(bp, _RET_IP_);
332
333 ASSERT(list_empty(&bp->b_lru));
334
335 if (bp->b_flags & (_XBF_PAGE_CACHE|_XBF_PAGES)) {
336 uint i;
337
338 if (xfs_buf_is_vmapped(bp))
339 vm_unmap_ram(bp->b_addr - bp->b_offset,
340 bp->b_page_count);
341
342 for (i = 0; i < bp->b_page_count; i++) {
343 struct page *page = bp->b_pages[i];
344
345 if (bp->b_flags & _XBF_PAGE_CACHE)
346 ASSERT(!PagePrivate(page));
347 page_cache_release(page);
348 }
349 }
350 _xfs_buf_free_pages(bp);
351 xfs_buf_deallocate(bp);
352}
353
354
355
356
357STATIC int
358_xfs_buf_lookup_pages(
359 xfs_buf_t *bp,
360 uint flags)
361{
362 struct address_space *mapping = bp->b_target->bt_mapping;
363 size_t blocksize = bp->b_target->bt_bsize;
364 size_t size = bp->b_count_desired;
365 size_t nbytes, offset;
366 gfp_t gfp_mask = xb_to_gfp(flags);
367 unsigned short page_count, i;
368 pgoff_t first;
369 xfs_off_t end;
370 int error;
371
372 end = bp->b_file_offset + bp->b_buffer_length;
373 page_count = xfs_buf_btoc(end) - xfs_buf_btoct(bp->b_file_offset);
374
375 error = _xfs_buf_get_pages(bp, page_count, flags);
376 if (unlikely(error))
377 return error;
378 bp->b_flags |= _XBF_PAGE_CACHE;
379
380 offset = bp->b_offset;
381 first = bp->b_file_offset >> PAGE_CACHE_SHIFT;
382
383 for (i = 0; i < bp->b_page_count; i++) {
384 struct page *page;
385 uint retries = 0;
386
387 retry:
388 page = find_or_create_page(mapping, first + i, gfp_mask);
389 if (unlikely(page == NULL)) {
390 if (flags & XBF_READ_AHEAD) {
391 bp->b_page_count = i;
392 for (i = 0; i < bp->b_page_count; i++)
393 unlock_page(bp->b_pages[i]);
394 return -ENOMEM;
395 }
396
397
398
399
400
401
402
403 if (!(++retries % 100))
404 printk(KERN_ERR
405 "XFS: possible memory allocation "
406 "deadlock in %s (mode:0x%x)\n",
407 __func__, gfp_mask);
408
409 XFS_STATS_INC(xb_page_retries);
410 congestion_wait(BLK_RW_ASYNC, HZ/50);
411 goto retry;
412 }
413
414 XFS_STATS_INC(xb_page_found);
415
416 nbytes = min_t(size_t, size, PAGE_CACHE_SIZE - offset);
417 size -= nbytes;
418
419 ASSERT(!PagePrivate(page));
420 if (!PageUptodate(page)) {
421 page_count--;
422 if (blocksize >= PAGE_CACHE_SIZE) {
423 if (flags & XBF_READ)
424 bp->b_flags |= _XBF_PAGE_LOCKED;
425 } else if (!PagePrivate(page)) {
426 if (test_page_region(page, offset, nbytes))
427 page_count++;
428 }
429 }
430
431 bp->b_pages[i] = page;
432 offset = 0;
433 }
434
435 if (!(bp->b_flags & _XBF_PAGE_LOCKED)) {
436 for (i = 0; i < bp->b_page_count; i++)
437 unlock_page(bp->b_pages[i]);
438 }
439
440 if (page_count == bp->b_page_count)
441 bp->b_flags |= XBF_DONE;
442
443 return error;
444}
445
446
447
448
449STATIC int
450_xfs_buf_map_pages(
451 xfs_buf_t *bp,
452 uint flags)
453{
454
455 if (bp->b_page_count == 1) {
456 bp->b_addr = page_address(bp->b_pages[0]) + bp->b_offset;
457 bp->b_flags |= XBF_MAPPED;
458 } else if (flags & XBF_MAPPED) {
459 bp->b_addr = vm_map_ram(bp->b_pages, bp->b_page_count,
460 -1, PAGE_KERNEL);
461 if (unlikely(bp->b_addr == NULL))
462 return -ENOMEM;
463 bp->b_addr += bp->b_offset;
464 bp->b_flags |= XBF_MAPPED;
465 }
466
467 return 0;
468}
469
470
471
472
473
474
475
476
477
478
479
480
481
482xfs_buf_t *
483_xfs_buf_find(
484 xfs_buftarg_t *btp,
485 xfs_off_t ioff,
486 size_t isize,
487 xfs_buf_flags_t flags,
488 xfs_buf_t *new_bp)
489{
490 xfs_off_t range_base;
491 size_t range_length;
492 struct xfs_perag *pag;
493 struct rb_node **rbp;
494 struct rb_node *parent;
495 xfs_buf_t *bp;
496
497 range_base = (ioff << BBSHIFT);
498 range_length = (isize << BBSHIFT);
499
500
501 ASSERT(!(range_length < (1 << btp->bt_sshift)));
502 ASSERT(!(range_base & (xfs_off_t)btp->bt_smask));
503
504
505 pag = xfs_perag_get(btp->bt_mount,
506 xfs_daddr_to_agno(btp->bt_mount, ioff));
507
508
509 spin_lock(&pag->pag_buf_lock);
510 rbp = &pag->pag_buf_tree.rb_node;
511 parent = NULL;
512 bp = NULL;
513 while (*rbp) {
514 parent = *rbp;
515 bp = rb_entry(parent, struct xfs_buf, b_rbnode);
516
517 if (range_base < bp->b_file_offset)
518 rbp = &(*rbp)->rb_left;
519 else if (range_base > bp->b_file_offset)
520 rbp = &(*rbp)->rb_right;
521 else {
522
523
524
525
526
527
528
529
530 if (bp->b_buffer_length != range_length) {
531 ASSERT(bp->b_flags & XBF_STALE);
532 rbp = &(*rbp)->rb_right;
533 continue;
534 }
535 atomic_inc(&bp->b_hold);
536 goto found;
537 }
538 }
539
540
541 if (new_bp) {
542 _xfs_buf_initialize(new_bp, btp, range_base,
543 range_length, flags);
544 rb_link_node(&new_bp->b_rbnode, parent, rbp);
545 rb_insert_color(&new_bp->b_rbnode, &pag->pag_buf_tree);
546
547 new_bp->b_pag = pag;
548 spin_unlock(&pag->pag_buf_lock);
549 } else {
550 XFS_STATS_INC(xb_miss_locked);
551 spin_unlock(&pag->pag_buf_lock);
552 xfs_perag_put(pag);
553 }
554 return new_bp;
555
556found:
557 spin_unlock(&pag->pag_buf_lock);
558 xfs_perag_put(pag);
559
560 if (xfs_buf_cond_lock(bp)) {
561
562 if (!(flags & XBF_TRYLOCK)) {
563 xfs_buf_lock(bp);
564 XFS_STATS_INC(xb_get_locked_waited);
565 } else {
566 xfs_buf_rele(bp);
567 XFS_STATS_INC(xb_busy_locked);
568 return NULL;
569 }
570 }
571
572 if (bp->b_flags & XBF_STALE) {
573 ASSERT((bp->b_flags & _XBF_DELWRI_Q) == 0);
574 bp->b_flags &= XBF_MAPPED;
575 }
576
577 trace_xfs_buf_find(bp, flags, _RET_IP_);
578 XFS_STATS_INC(xb_get_locked);
579 return bp;
580}
581
582
583
584
585
586
587xfs_buf_t *
588xfs_buf_get(
589 xfs_buftarg_t *target,
590 xfs_off_t ioff,
591 size_t isize,
592 xfs_buf_flags_t flags)
593{
594 xfs_buf_t *bp, *new_bp;
595 int error = 0, i;
596
597 new_bp = xfs_buf_allocate(flags);
598 if (unlikely(!new_bp))
599 return NULL;
600
601 bp = _xfs_buf_find(target, ioff, isize, flags, new_bp);
602 if (bp == new_bp) {
603 error = _xfs_buf_lookup_pages(bp, flags);
604 if (error)
605 goto no_buffer;
606 } else {
607 xfs_buf_deallocate(new_bp);
608 if (unlikely(bp == NULL))
609 return NULL;
610 }
611
612 for (i = 0; i < bp->b_page_count; i++)
613 mark_page_accessed(bp->b_pages[i]);
614
615 if (!(bp->b_flags & XBF_MAPPED)) {
616 error = _xfs_buf_map_pages(bp, flags);
617 if (unlikely(error)) {
618 printk(KERN_WARNING "%s: failed to map pages\n",
619 __func__);
620 goto no_buffer;
621 }
622 }
623
624 XFS_STATS_INC(xb_get);
625
626
627
628
629
630 bp->b_bn = ioff;
631 bp->b_count_desired = bp->b_buffer_length;
632
633 trace_xfs_buf_get(bp, flags, _RET_IP_);
634 return bp;
635
636 no_buffer:
637 if (flags & (XBF_LOCK | XBF_TRYLOCK))
638 xfs_buf_unlock(bp);
639 xfs_buf_rele(bp);
640 return NULL;
641}
642
643STATIC int
644_xfs_buf_read(
645 xfs_buf_t *bp,
646 xfs_buf_flags_t flags)
647{
648 int status;
649
650 ASSERT(!(flags & (XBF_DELWRI|XBF_WRITE)));
651 ASSERT(bp->b_bn != XFS_BUF_DADDR_NULL);
652
653 bp->b_flags &= ~(XBF_WRITE | XBF_ASYNC | XBF_DELWRI | \
654 XBF_READ_AHEAD | _XBF_RUN_QUEUES);
655 bp->b_flags |= flags & (XBF_READ | XBF_ASYNC | \
656 XBF_READ_AHEAD | _XBF_RUN_QUEUES);
657
658 status = xfs_buf_iorequest(bp);
659 if (status || XFS_BUF_ISERROR(bp) || (flags & XBF_ASYNC))
660 return status;
661 return xfs_buf_iowait(bp);
662}
663
664xfs_buf_t *
665xfs_buf_read(
666 xfs_buftarg_t *target,
667 xfs_off_t ioff,
668 size_t isize,
669 xfs_buf_flags_t flags)
670{
671 xfs_buf_t *bp;
672
673 flags |= XBF_READ;
674
675 bp = xfs_buf_get(target, ioff, isize, flags);
676 if (bp) {
677 trace_xfs_buf_read(bp, flags, _RET_IP_);
678
679 if (!XFS_BUF_ISDONE(bp)) {
680 XFS_STATS_INC(xb_get_read);
681 _xfs_buf_read(bp, flags);
682 } else if (flags & XBF_ASYNC) {
683
684
685
686
687 goto no_buffer;
688 } else {
689
690 bp->b_flags &= ~XBF_READ;
691 }
692 }
693
694 return bp;
695
696 no_buffer:
697 if (flags & (XBF_LOCK | XBF_TRYLOCK))
698 xfs_buf_unlock(bp);
699 xfs_buf_rele(bp);
700 return NULL;
701}
702
703
704
705
706
707void
708xfs_buf_readahead(
709 xfs_buftarg_t *target,
710 xfs_off_t ioff,
711 size_t isize)
712{
713 struct backing_dev_info *bdi;
714
715 bdi = target->bt_mapping->backing_dev_info;
716 if (bdi_read_congested(bdi))
717 return;
718
719 xfs_buf_read(target, ioff, isize,
720 XBF_TRYLOCK|XBF_ASYNC|XBF_READ_AHEAD|XBF_DONT_BLOCK);
721}
722
723
724
725
726
727struct xfs_buf *
728xfs_buf_read_uncached(
729 struct xfs_mount *mp,
730 struct xfs_buftarg *target,
731 xfs_daddr_t daddr,
732 size_t length,
733 int flags)
734{
735 xfs_buf_t *bp;
736 int error;
737
738 bp = xfs_buf_get_uncached(target, length, flags);
739 if (!bp)
740 return NULL;
741
742
743 xfs_buf_lock(bp);
744 XFS_BUF_SET_ADDR(bp, daddr);
745 XFS_BUF_READ(bp);
746 XFS_BUF_BUSY(bp);
747
748 xfsbdstrat(mp, bp);
749 error = xfs_buf_iowait(bp);
750 if (error || bp->b_error) {
751 xfs_buf_relse(bp);
752 return NULL;
753 }
754 return bp;
755}
756
757xfs_buf_t *
758xfs_buf_get_empty(
759 size_t len,
760 xfs_buftarg_t *target)
761{
762 xfs_buf_t *bp;
763
764 bp = xfs_buf_allocate(0);
765 if (bp)
766 _xfs_buf_initialize(bp, target, 0, len, 0);
767 return bp;
768}
769
770static inline struct page *
771mem_to_page(
772 void *addr)
773{
774 if ((!is_vmalloc_addr(addr))) {
775 return virt_to_page(addr);
776 } else {
777 return vmalloc_to_page(addr);
778 }
779}
780
781int
782xfs_buf_associate_memory(
783 xfs_buf_t *bp,
784 void *mem,
785 size_t len)
786{
787 int rval;
788 int i = 0;
789 unsigned long pageaddr;
790 unsigned long offset;
791 size_t buflen;
792 int page_count;
793
794 pageaddr = (unsigned long)mem & PAGE_CACHE_MASK;
795 offset = (unsigned long)mem - pageaddr;
796 buflen = PAGE_CACHE_ALIGN(len + offset);
797 page_count = buflen >> PAGE_CACHE_SHIFT;
798
799
800 if (bp->b_pages)
801 _xfs_buf_free_pages(bp);
802
803 bp->b_pages = NULL;
804 bp->b_addr = mem;
805
806 rval = _xfs_buf_get_pages(bp, page_count, XBF_DONT_BLOCK);
807 if (rval)
808 return rval;
809
810 bp->b_offset = offset;
811
812 for (i = 0; i < bp->b_page_count; i++) {
813 bp->b_pages[i] = mem_to_page((void *)pageaddr);
814 pageaddr += PAGE_CACHE_SIZE;
815 }
816
817 bp->b_count_desired = len;
818 bp->b_buffer_length = buflen;
819 bp->b_flags |= XBF_MAPPED;
820 bp->b_flags &= ~_XBF_PAGE_LOCKED;
821
822 return 0;
823}
824
825xfs_buf_t *
826xfs_buf_get_uncached(
827 struct xfs_buftarg *target,
828 size_t len,
829 int flags)
830{
831 unsigned long page_count = PAGE_ALIGN(len) >> PAGE_SHIFT;
832 int error, i;
833 xfs_buf_t *bp;
834
835 bp = xfs_buf_allocate(0);
836 if (unlikely(bp == NULL))
837 goto fail;
838 _xfs_buf_initialize(bp, target, 0, len, 0);
839
840 error = _xfs_buf_get_pages(bp, page_count, 0);
841 if (error)
842 goto fail_free_buf;
843
844 for (i = 0; i < page_count; i++) {
845 bp->b_pages[i] = alloc_page(xb_to_gfp(flags));
846 if (!bp->b_pages[i])
847 goto fail_free_mem;
848 }
849 bp->b_flags |= _XBF_PAGES;
850
851 error = _xfs_buf_map_pages(bp, XBF_MAPPED);
852 if (unlikely(error)) {
853 printk(KERN_WARNING "%s: failed to map pages\n",
854 __func__);
855 goto fail_free_mem;
856 }
857
858 xfs_buf_unlock(bp);
859
860 trace_xfs_buf_get_uncached(bp, _RET_IP_);
861 return bp;
862
863 fail_free_mem:
864 while (--i >= 0)
865 __free_page(bp->b_pages[i]);
866 _xfs_buf_free_pages(bp);
867 fail_free_buf:
868 xfs_buf_deallocate(bp);
869 fail:
870 return NULL;
871}
872
873
874
875
876
877
878void
879xfs_buf_hold(
880 xfs_buf_t *bp)
881{
882 trace_xfs_buf_hold(bp, _RET_IP_);
883 atomic_inc(&bp->b_hold);
884}
885
886
887
888
889
890void
891xfs_buf_rele(
892 xfs_buf_t *bp)
893{
894 struct xfs_perag *pag = bp->b_pag;
895
896 trace_xfs_buf_rele(bp, _RET_IP_);
897
898 if (!pag) {
899 ASSERT(list_empty(&bp->b_lru));
900 ASSERT(RB_EMPTY_NODE(&bp->b_rbnode));
901 if (atomic_dec_and_test(&bp->b_hold))
902 xfs_buf_free(bp);
903 return;
904 }
905
906 ASSERT(!RB_EMPTY_NODE(&bp->b_rbnode));
907
908 ASSERT(atomic_read(&bp->b_hold) > 0);
909 if (atomic_dec_and_lock(&bp->b_hold, &pag->pag_buf_lock)) {
910 if (!(bp->b_flags & XBF_STALE) &&
911 atomic_read(&bp->b_lru_ref)) {
912 xfs_buf_lru_add(bp);
913 spin_unlock(&pag->pag_buf_lock);
914 } else {
915 xfs_buf_lru_del(bp);
916 ASSERT(!(bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)));
917 rb_erase(&bp->b_rbnode, &pag->pag_buf_tree);
918 spin_unlock(&pag->pag_buf_lock);
919 xfs_perag_put(pag);
920 xfs_buf_free(bp);
921 }
922 }
923}
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950int
951xfs_buf_cond_lock(
952 xfs_buf_t *bp)
953{
954 int locked;
955
956 locked = down_trylock(&bp->b_sema) == 0;
957 if (locked)
958 XB_SET_OWNER(bp);
959 else if (atomic_read(&bp->b_pin_count) && (bp->b_flags & XBF_STALE))
960 xfs_log_force(bp->b_target->bt_mount, 0);
961
962 trace_xfs_buf_cond_lock(bp, _RET_IP_);
963 return locked ? 0 : -EBUSY;
964}
965
966int
967xfs_buf_lock_value(
968 xfs_buf_t *bp)
969{
970 return bp->b_sema.count;
971}
972
973
974
975
976
977
978
979
980
981
982
983
984
985void
986xfs_buf_lock(
987 xfs_buf_t *bp)
988{
989 trace_xfs_buf_lock(bp, _RET_IP_);
990
991 if (atomic_read(&bp->b_pin_count) && (bp->b_flags & XBF_STALE))
992 xfs_log_force(bp->b_target->bt_mount, 0);
993 if (atomic_read(&bp->b_io_remaining))
994 blk_run_address_space(bp->b_target->bt_mapping);
995 down(&bp->b_sema);
996 XB_SET_OWNER(bp);
997
998 trace_xfs_buf_lock_done(bp, _RET_IP_);
999}
1000
1001
1002
1003
1004
1005
1006
1007
1008void
1009xfs_buf_unlock(
1010 xfs_buf_t *bp)
1011{
1012 if ((bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)) == XBF_DELWRI) {
1013 atomic_inc(&bp->b_hold);
1014 bp->b_flags |= XBF_ASYNC;
1015 xfs_buf_delwri_queue(bp, 0);
1016 }
1017
1018 XB_CLEAR_OWNER(bp);
1019 up(&bp->b_sema);
1020
1021 trace_xfs_buf_unlock(bp, _RET_IP_);
1022}
1023
1024STATIC void
1025xfs_buf_wait_unpin(
1026 xfs_buf_t *bp)
1027{
1028 DECLARE_WAITQUEUE (wait, current);
1029
1030 if (atomic_read(&bp->b_pin_count) == 0)
1031 return;
1032
1033 add_wait_queue(&bp->b_waiters, &wait);
1034 for (;;) {
1035 set_current_state(TASK_UNINTERRUPTIBLE);
1036 if (atomic_read(&bp->b_pin_count) == 0)
1037 break;
1038 if (atomic_read(&bp->b_io_remaining))
1039 blk_run_address_space(bp->b_target->bt_mapping);
1040 schedule();
1041 }
1042 remove_wait_queue(&bp->b_waiters, &wait);
1043 set_current_state(TASK_RUNNING);
1044}
1045
1046
1047
1048
1049
1050STATIC void
1051xfs_buf_iodone_work(
1052 struct work_struct *work)
1053{
1054 xfs_buf_t *bp =
1055 container_of(work, xfs_buf_t, b_iodone_work);
1056
1057 if (bp->b_iodone)
1058 (*(bp->b_iodone))(bp);
1059 else if (bp->b_flags & XBF_ASYNC)
1060 xfs_buf_relse(bp);
1061}
1062
1063void
1064xfs_buf_ioend(
1065 xfs_buf_t *bp,
1066 int schedule)
1067{
1068 trace_xfs_buf_iodone(bp, _RET_IP_);
1069
1070 bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_READ_AHEAD);
1071 if (bp->b_error == 0)
1072 bp->b_flags |= XBF_DONE;
1073
1074 if ((bp->b_iodone) || (bp->b_flags & XBF_ASYNC)) {
1075 if (schedule) {
1076 INIT_WORK(&bp->b_iodone_work, xfs_buf_iodone_work);
1077 queue_work(xfslogd_workqueue, &bp->b_iodone_work);
1078 } else {
1079 xfs_buf_iodone_work(&bp->b_iodone_work);
1080 }
1081 } else {
1082 complete(&bp->b_iowait);
1083 }
1084}
1085
1086void
1087xfs_buf_ioerror(
1088 xfs_buf_t *bp,
1089 int error)
1090{
1091 ASSERT(error >= 0 && error <= 0xffff);
1092 bp->b_error = (unsigned short)error;
1093 trace_xfs_buf_ioerror(bp, error, _RET_IP_);
1094}
1095
1096int
1097xfs_bwrite(
1098 struct xfs_mount *mp,
1099 struct xfs_buf *bp)
1100{
1101 int error;
1102
1103 bp->b_flags |= XBF_WRITE;
1104 bp->b_flags &= ~(XBF_ASYNC | XBF_READ);
1105
1106 xfs_buf_delwri_dequeue(bp);
1107 xfs_bdstrat_cb(bp);
1108
1109 error = xfs_buf_iowait(bp);
1110 if (error)
1111 xfs_force_shutdown(mp, SHUTDOWN_META_IO_ERROR);
1112 xfs_buf_relse(bp);
1113 return error;
1114}
1115
1116void
1117xfs_bdwrite(
1118 void *mp,
1119 struct xfs_buf *bp)
1120{
1121 trace_xfs_buf_bdwrite(bp, _RET_IP_);
1122
1123 bp->b_flags &= ~XBF_READ;
1124 bp->b_flags |= (XBF_DELWRI | XBF_ASYNC);
1125
1126 xfs_buf_delwri_queue(bp, 1);
1127}
1128
1129
1130
1131
1132
1133
1134STATIC int
1135xfs_bioerror(
1136 xfs_buf_t *bp)
1137{
1138#ifdef XFSERRORDEBUG
1139 ASSERT(XFS_BUF_ISREAD(bp) || bp->b_iodone);
1140#endif
1141
1142
1143
1144
1145 XFS_BUF_ERROR(bp, EIO);
1146
1147
1148
1149
1150 XFS_BUF_UNREAD(bp);
1151 XFS_BUF_UNDELAYWRITE(bp);
1152 XFS_BUF_UNDONE(bp);
1153 XFS_BUF_STALE(bp);
1154
1155 xfs_buf_ioend(bp, 0);
1156
1157 return EIO;
1158}
1159
1160
1161
1162
1163
1164
1165
1166STATIC int
1167xfs_bioerror_relse(
1168 struct xfs_buf *bp)
1169{
1170 int64_t fl = XFS_BUF_BFLAGS(bp);
1171
1172
1173
1174
1175
1176
1177
1178
1179 XFS_BUF_UNREAD(bp);
1180 XFS_BUF_UNDELAYWRITE(bp);
1181 XFS_BUF_DONE(bp);
1182 XFS_BUF_STALE(bp);
1183 XFS_BUF_CLR_IODONE_FUNC(bp);
1184 if (!(fl & XBF_ASYNC)) {
1185
1186
1187
1188
1189
1190
1191 XFS_BUF_ERROR(bp, EIO);
1192 XFS_BUF_FINISH_IOWAIT(bp);
1193 } else {
1194 xfs_buf_relse(bp);
1195 }
1196
1197 return EIO;
1198}
1199
1200
1201
1202
1203
1204
1205
1206
1207int
1208xfs_bdstrat_cb(
1209 struct xfs_buf *bp)
1210{
1211 if (XFS_FORCED_SHUTDOWN(bp->b_target->bt_mount)) {
1212 trace_xfs_bdstrat_shut(bp, _RET_IP_);
1213
1214
1215
1216
1217
1218 if (!bp->b_iodone && !XFS_BUF_ISREAD(bp))
1219 return xfs_bioerror_relse(bp);
1220 else
1221 return xfs_bioerror(bp);
1222 }
1223
1224 xfs_buf_iorequest(bp);
1225 return 0;
1226}
1227
1228
1229
1230
1231
1232
1233void
1234xfsbdstrat(
1235 struct xfs_mount *mp,
1236 struct xfs_buf *bp)
1237{
1238 if (XFS_FORCED_SHUTDOWN(mp)) {
1239 trace_xfs_bdstrat_shut(bp, _RET_IP_);
1240 xfs_bioerror_relse(bp);
1241 return;
1242 }
1243
1244 xfs_buf_iorequest(bp);
1245}
1246
1247STATIC void
1248_xfs_buf_ioend(
1249 xfs_buf_t *bp,
1250 int schedule)
1251{
1252 if (atomic_dec_and_test(&bp->b_io_remaining) == 1) {
1253 bp->b_flags &= ~_XBF_PAGE_LOCKED;
1254 xfs_buf_ioend(bp, schedule);
1255 }
1256}
1257
1258STATIC void
1259xfs_buf_bio_end_io(
1260 struct bio *bio,
1261 int error)
1262{
1263 xfs_buf_t *bp = (xfs_buf_t *)bio->bi_private;
1264 unsigned int blocksize = bp->b_target->bt_bsize;
1265 struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
1266
1267 xfs_buf_ioerror(bp, -error);
1268
1269 if (!error && xfs_buf_is_vmapped(bp) && (bp->b_flags & XBF_READ))
1270 invalidate_kernel_vmap_range(bp->b_addr, xfs_buf_vmap_len(bp));
1271
1272 do {
1273 struct page *page = bvec->bv_page;
1274
1275 ASSERT(!PagePrivate(page));
1276 if (unlikely(bp->b_error)) {
1277 if (bp->b_flags & XBF_READ)
1278 ClearPageUptodate(page);
1279 } else if (blocksize >= PAGE_CACHE_SIZE) {
1280 SetPageUptodate(page);
1281 } else if (!PagePrivate(page) &&
1282 (bp->b_flags & _XBF_PAGE_CACHE)) {
1283 set_page_region(page, bvec->bv_offset, bvec->bv_len);
1284 }
1285
1286 if (--bvec >= bio->bi_io_vec)
1287 prefetchw(&bvec->bv_page->flags);
1288
1289 if (bp->b_flags & _XBF_PAGE_LOCKED)
1290 unlock_page(page);
1291 } while (bvec >= bio->bi_io_vec);
1292
1293 _xfs_buf_ioend(bp, 1);
1294 bio_put(bio);
1295}
1296
1297STATIC void
1298_xfs_buf_ioapply(
1299 xfs_buf_t *bp)
1300{
1301 int rw, map_i, total_nr_pages, nr_pages;
1302 struct bio *bio;
1303 int offset = bp->b_offset;
1304 int size = bp->b_count_desired;
1305 sector_t sector = bp->b_bn;
1306 unsigned int blocksize = bp->b_target->bt_bsize;
1307
1308 total_nr_pages = bp->b_page_count;
1309 map_i = 0;
1310
1311 if (bp->b_flags & XBF_ORDERED) {
1312 ASSERT(!(bp->b_flags & XBF_READ));
1313 rw = WRITE_FLUSH_FUA;
1314 } else if (bp->b_flags & XBF_LOG_BUFFER) {
1315 ASSERT(!(bp->b_flags & XBF_READ_AHEAD));
1316 bp->b_flags &= ~_XBF_RUN_QUEUES;
1317 rw = (bp->b_flags & XBF_WRITE) ? WRITE_SYNC : READ_SYNC;
1318 } else if (bp->b_flags & _XBF_RUN_QUEUES) {
1319 ASSERT(!(bp->b_flags & XBF_READ_AHEAD));
1320 bp->b_flags &= ~_XBF_RUN_QUEUES;
1321 rw = (bp->b_flags & XBF_WRITE) ? WRITE_META : READ_META;
1322 } else {
1323 rw = (bp->b_flags & XBF_WRITE) ? WRITE :
1324 (bp->b_flags & XBF_READ_AHEAD) ? READA : READ;
1325 }
1326
1327
1328
1329
1330
1331
1332 if ((bp->b_buffer_length < PAGE_CACHE_SIZE) &&
1333 ((bp->b_flags & (XBF_READ|_XBF_PAGE_LOCKED)) ==
1334 (XBF_READ|_XBF_PAGE_LOCKED)) &&
1335 (blocksize >= PAGE_CACHE_SIZE)) {
1336 bio = bio_alloc(GFP_NOIO, 1);
1337
1338 bio->bi_bdev = bp->b_target->bt_bdev;
1339 bio->bi_sector = sector - (offset >> BBSHIFT);
1340 bio->bi_end_io = xfs_buf_bio_end_io;
1341 bio->bi_private = bp;
1342
1343 bio_add_page(bio, bp->b_pages[0], PAGE_CACHE_SIZE, 0);
1344 size = 0;
1345
1346 atomic_inc(&bp->b_io_remaining);
1347
1348 goto submit_io;
1349 }
1350
1351next_chunk:
1352 atomic_inc(&bp->b_io_remaining);
1353 nr_pages = BIO_MAX_SECTORS >> (PAGE_SHIFT - BBSHIFT);
1354 if (nr_pages > total_nr_pages)
1355 nr_pages = total_nr_pages;
1356
1357 bio = bio_alloc(GFP_NOIO, nr_pages);
1358 bio->bi_bdev = bp->b_target->bt_bdev;
1359 bio->bi_sector = sector;
1360 bio->bi_end_io = xfs_buf_bio_end_io;
1361 bio->bi_private = bp;
1362
1363 for (; size && nr_pages; nr_pages--, map_i++) {
1364 int rbytes, nbytes = PAGE_CACHE_SIZE - offset;
1365
1366 if (nbytes > size)
1367 nbytes = size;
1368
1369 rbytes = bio_add_page(bio, bp->b_pages[map_i], nbytes, offset);
1370 if (rbytes < nbytes)
1371 break;
1372
1373 offset = 0;
1374 sector += nbytes >> BBSHIFT;
1375 size -= nbytes;
1376 total_nr_pages--;
1377 }
1378
1379submit_io:
1380 if (likely(bio->bi_size)) {
1381 if (xfs_buf_is_vmapped(bp)) {
1382 flush_kernel_vmap_range(bp->b_addr,
1383 xfs_buf_vmap_len(bp));
1384 }
1385 submit_bio(rw, bio);
1386 if (size)
1387 goto next_chunk;
1388 } else {
1389
1390
1391
1392
1393
1394
1395 xfs_buf_ioerror(bp, EIO);
1396 if (bp->b_flags & _XBF_PAGE_LOCKED) {
1397 int i;
1398 for (i = 0; i < bp->b_page_count; i++)
1399 unlock_page(bp->b_pages[i]);
1400 }
1401 bio_put(bio);
1402 }
1403}
1404
1405int
1406xfs_buf_iorequest(
1407 xfs_buf_t *bp)
1408{
1409 trace_xfs_buf_iorequest(bp, _RET_IP_);
1410
1411 if (bp->b_flags & XBF_DELWRI) {
1412 xfs_buf_delwri_queue(bp, 1);
1413 return 0;
1414 }
1415
1416 if (bp->b_flags & XBF_WRITE) {
1417 xfs_buf_wait_unpin(bp);
1418 }
1419
1420 xfs_buf_hold(bp);
1421
1422
1423
1424
1425
1426 atomic_set(&bp->b_io_remaining, 1);
1427 _xfs_buf_ioapply(bp);
1428 _xfs_buf_ioend(bp, 0);
1429
1430 xfs_buf_rele(bp);
1431 return 0;
1432}
1433
1434
1435
1436
1437
1438
1439int
1440xfs_buf_iowait(
1441 xfs_buf_t *bp)
1442{
1443 trace_xfs_buf_iowait(bp, _RET_IP_);
1444
1445 if (atomic_read(&bp->b_io_remaining))
1446 blk_run_address_space(bp->b_target->bt_mapping);
1447 wait_for_completion(&bp->b_iowait);
1448
1449 trace_xfs_buf_iowait_done(bp, _RET_IP_);
1450 return bp->b_error;
1451}
1452
1453xfs_caddr_t
1454xfs_buf_offset(
1455 xfs_buf_t *bp,
1456 size_t offset)
1457{
1458 struct page *page;
1459
1460 if (bp->b_flags & XBF_MAPPED)
1461 return XFS_BUF_PTR(bp) + offset;
1462
1463 offset += bp->b_offset;
1464 page = bp->b_pages[offset >> PAGE_CACHE_SHIFT];
1465 return (xfs_caddr_t)page_address(page) + (offset & (PAGE_CACHE_SIZE-1));
1466}
1467
1468
1469
1470
1471void
1472xfs_buf_iomove(
1473 xfs_buf_t *bp,
1474 size_t boff,
1475 size_t bsize,
1476 void *data,
1477 xfs_buf_rw_t mode)
1478{
1479 size_t bend, cpoff, csize;
1480 struct page *page;
1481
1482 bend = boff + bsize;
1483 while (boff < bend) {
1484 page = bp->b_pages[xfs_buf_btoct(boff + bp->b_offset)];
1485 cpoff = xfs_buf_poff(boff + bp->b_offset);
1486 csize = min_t(size_t,
1487 PAGE_CACHE_SIZE-cpoff, bp->b_count_desired-boff);
1488
1489 ASSERT(((csize + cpoff) <= PAGE_CACHE_SIZE));
1490
1491 switch (mode) {
1492 case XBRW_ZERO:
1493 memset(page_address(page) + cpoff, 0, csize);
1494 break;
1495 case XBRW_READ:
1496 memcpy(data, page_address(page) + cpoff, csize);
1497 break;
1498 case XBRW_WRITE:
1499 memcpy(page_address(page) + cpoff, data, csize);
1500 }
1501
1502 boff += csize;
1503 data += csize;
1504 }
1505}
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516void
1517xfs_wait_buftarg(
1518 struct xfs_buftarg *btp)
1519{
1520 struct xfs_buf *bp;
1521
1522restart:
1523 spin_lock(&btp->bt_lru_lock);
1524 while (!list_empty(&btp->bt_lru)) {
1525 bp = list_first_entry(&btp->bt_lru, struct xfs_buf, b_lru);
1526 if (atomic_read(&bp->b_hold) > 1) {
1527 spin_unlock(&btp->bt_lru_lock);
1528 delay(100);
1529 goto restart;
1530 }
1531
1532
1533
1534
1535 atomic_set(&bp->b_lru_ref, 0);
1536 spin_unlock(&btp->bt_lru_lock);
1537 xfs_buf_rele(bp);
1538 spin_lock(&btp->bt_lru_lock);
1539 }
1540 spin_unlock(&btp->bt_lru_lock);
1541}
1542
1543int
1544xfs_buftarg_shrink(
1545 struct shrinker *shrink,
1546 int nr_to_scan,
1547 gfp_t mask)
1548{
1549 struct xfs_buftarg *btp = container_of(shrink,
1550 struct xfs_buftarg, bt_shrinker);
1551 struct xfs_buf *bp;
1552 LIST_HEAD(dispose);
1553
1554 if (!nr_to_scan)
1555 return btp->bt_lru_nr;
1556
1557 spin_lock(&btp->bt_lru_lock);
1558 while (!list_empty(&btp->bt_lru)) {
1559 if (nr_to_scan-- <= 0)
1560 break;
1561
1562 bp = list_first_entry(&btp->bt_lru, struct xfs_buf, b_lru);
1563
1564
1565
1566
1567
1568
1569 if (!atomic_add_unless(&bp->b_lru_ref, -1, 0)) {
1570 list_move_tail(&bp->b_lru, &btp->bt_lru);
1571 continue;
1572 }
1573
1574
1575
1576
1577
1578 list_move(&bp->b_lru, &dispose);
1579 btp->bt_lru_nr--;
1580 }
1581 spin_unlock(&btp->bt_lru_lock);
1582
1583 while (!list_empty(&dispose)) {
1584 bp = list_first_entry(&dispose, struct xfs_buf, b_lru);
1585 list_del_init(&bp->b_lru);
1586 xfs_buf_rele(bp);
1587 }
1588
1589 return btp->bt_lru_nr;
1590}
1591
1592void
1593xfs_free_buftarg(
1594 struct xfs_mount *mp,
1595 struct xfs_buftarg *btp)
1596{
1597 unregister_shrinker(&btp->bt_shrinker);
1598
1599 xfs_flush_buftarg(btp, 1);
1600 if (mp->m_flags & XFS_MOUNT_BARRIER)
1601 xfs_blkdev_issue_flush(btp);
1602 iput(btp->bt_mapping->host);
1603
1604 kthread_stop(btp->bt_task);
1605 kmem_free(btp);
1606}
1607
1608STATIC int
1609xfs_setsize_buftarg_flags(
1610 xfs_buftarg_t *btp,
1611 unsigned int blocksize,
1612 unsigned int sectorsize,
1613 int verbose)
1614{
1615 btp->bt_bsize = blocksize;
1616 btp->bt_sshift = ffs(sectorsize) - 1;
1617 btp->bt_smask = sectorsize - 1;
1618
1619 if (set_blocksize(btp->bt_bdev, sectorsize)) {
1620 printk(KERN_WARNING
1621 "XFS: Cannot set_blocksize to %u on device %s\n",
1622 sectorsize, XFS_BUFTARG_NAME(btp));
1623 return EINVAL;
1624 }
1625
1626 if (verbose &&
1627 (PAGE_CACHE_SIZE / BITS_PER_LONG) > sectorsize) {
1628 printk(KERN_WARNING
1629 "XFS: %u byte sectors in use on device %s. "
1630 "This is suboptimal; %u or greater is ideal.\n",
1631 sectorsize, XFS_BUFTARG_NAME(btp),
1632 (unsigned int)PAGE_CACHE_SIZE / BITS_PER_LONG);
1633 }
1634
1635 return 0;
1636}
1637
1638
1639
1640
1641
1642
1643STATIC int
1644xfs_setsize_buftarg_early(
1645 xfs_buftarg_t *btp,
1646 struct block_device *bdev)
1647{
1648 return xfs_setsize_buftarg_flags(btp,
1649 PAGE_CACHE_SIZE, bdev_logical_block_size(bdev), 0);
1650}
1651
1652int
1653xfs_setsize_buftarg(
1654 xfs_buftarg_t *btp,
1655 unsigned int blocksize,
1656 unsigned int sectorsize)
1657{
1658 return xfs_setsize_buftarg_flags(btp, blocksize, sectorsize, 1);
1659}
1660
1661STATIC int
1662xfs_mapping_buftarg(
1663 xfs_buftarg_t *btp,
1664 struct block_device *bdev)
1665{
1666 struct backing_dev_info *bdi;
1667 struct inode *inode;
1668 struct address_space *mapping;
1669 static const struct address_space_operations mapping_aops = {
1670 .sync_page = block_sync_page,
1671 .migratepage = fail_migrate_page,
1672 };
1673
1674 inode = new_inode(bdev->bd_inode->i_sb);
1675 if (!inode) {
1676 printk(KERN_WARNING
1677 "XFS: Cannot allocate mapping inode for device %s\n",
1678 XFS_BUFTARG_NAME(btp));
1679 return ENOMEM;
1680 }
1681 inode->i_ino = get_next_ino();
1682 inode->i_mode = S_IFBLK;
1683 inode->i_bdev = bdev;
1684 inode->i_rdev = bdev->bd_dev;
1685 bdi = blk_get_backing_dev_info(bdev);
1686 if (!bdi)
1687 bdi = &default_backing_dev_info;
1688 mapping = &inode->i_data;
1689 mapping->a_ops = &mapping_aops;
1690 mapping->backing_dev_info = bdi;
1691 mapping_set_gfp_mask(mapping, GFP_NOFS);
1692 btp->bt_mapping = mapping;
1693 return 0;
1694}
1695
1696STATIC int
1697xfs_alloc_delwrite_queue(
1698 xfs_buftarg_t *btp,
1699 const char *fsname)
1700{
1701 INIT_LIST_HEAD(&btp->bt_delwrite_queue);
1702 spin_lock_init(&btp->bt_delwrite_lock);
1703 btp->bt_flags = 0;
1704 btp->bt_task = kthread_run(xfsbufd, btp, "xfsbufd/%s", fsname);
1705 if (IS_ERR(btp->bt_task))
1706 return PTR_ERR(btp->bt_task);
1707 return 0;
1708}
1709
1710xfs_buftarg_t *
1711xfs_alloc_buftarg(
1712 struct xfs_mount *mp,
1713 struct block_device *bdev,
1714 int external,
1715 const char *fsname)
1716{
1717 xfs_buftarg_t *btp;
1718
1719 btp = kmem_zalloc(sizeof(*btp), KM_SLEEP);
1720
1721 btp->bt_mount = mp;
1722 btp->bt_dev = bdev->bd_dev;
1723 btp->bt_bdev = bdev;
1724 INIT_LIST_HEAD(&btp->bt_lru);
1725 spin_lock_init(&btp->bt_lru_lock);
1726 if (xfs_setsize_buftarg_early(btp, bdev))
1727 goto error;
1728 if (xfs_mapping_buftarg(btp, bdev))
1729 goto error;
1730 if (xfs_alloc_delwrite_queue(btp, fsname))
1731 goto error;
1732 btp->bt_shrinker.shrink = xfs_buftarg_shrink;
1733 btp->bt_shrinker.seeks = DEFAULT_SEEKS;
1734 register_shrinker(&btp->bt_shrinker);
1735 return btp;
1736
1737error:
1738 kmem_free(btp);
1739 return NULL;
1740}
1741
1742
1743
1744
1745
1746STATIC void
1747xfs_buf_delwri_queue(
1748 xfs_buf_t *bp,
1749 int unlock)
1750{
1751 struct list_head *dwq = &bp->b_target->bt_delwrite_queue;
1752 spinlock_t *dwlk = &bp->b_target->bt_delwrite_lock;
1753
1754 trace_xfs_buf_delwri_queue(bp, _RET_IP_);
1755
1756 ASSERT((bp->b_flags&(XBF_DELWRI|XBF_ASYNC)) == (XBF_DELWRI|XBF_ASYNC));
1757
1758 spin_lock(dwlk);
1759
1760 if (!list_empty(&bp->b_list)) {
1761 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1762 if (unlock)
1763 atomic_dec(&bp->b_hold);
1764 list_del(&bp->b_list);
1765 }
1766
1767 if (list_empty(dwq)) {
1768
1769 wake_up_process(bp->b_target->bt_task);
1770 }
1771
1772 bp->b_flags |= _XBF_DELWRI_Q;
1773 list_add_tail(&bp->b_list, dwq);
1774 bp->b_queuetime = jiffies;
1775 spin_unlock(dwlk);
1776
1777 if (unlock)
1778 xfs_buf_unlock(bp);
1779}
1780
1781void
1782xfs_buf_delwri_dequeue(
1783 xfs_buf_t *bp)
1784{
1785 spinlock_t *dwlk = &bp->b_target->bt_delwrite_lock;
1786 int dequeued = 0;
1787
1788 spin_lock(dwlk);
1789 if ((bp->b_flags & XBF_DELWRI) && !list_empty(&bp->b_list)) {
1790 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1791 list_del_init(&bp->b_list);
1792 dequeued = 1;
1793 }
1794 bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q);
1795 spin_unlock(dwlk);
1796
1797 if (dequeued)
1798 xfs_buf_rele(bp);
1799
1800 trace_xfs_buf_delwri_dequeue(bp, _RET_IP_);
1801}
1802
1803
1804
1805
1806
1807
1808
1809
1810void
1811xfs_buf_delwri_promote(
1812 struct xfs_buf *bp)
1813{
1814 struct xfs_buftarg *btp = bp->b_target;
1815 long age = xfs_buf_age_centisecs * msecs_to_jiffies(10) + 1;
1816
1817 ASSERT(bp->b_flags & XBF_DELWRI);
1818 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1819
1820
1821
1822
1823
1824 if (bp->b_queuetime < jiffies - age)
1825 return;
1826 bp->b_queuetime = jiffies - age;
1827 spin_lock(&btp->bt_delwrite_lock);
1828 list_move(&bp->b_list, &btp->bt_delwrite_queue);
1829 spin_unlock(&btp->bt_delwrite_lock);
1830}
1831
1832STATIC void
1833xfs_buf_runall_queues(
1834 struct workqueue_struct *queue)
1835{
1836 flush_workqueue(queue);
1837}
1838
1839
1840
1841
1842
1843STATIC int
1844xfs_buf_delwri_split(
1845 xfs_buftarg_t *target,
1846 struct list_head *list,
1847 unsigned long age)
1848{
1849 xfs_buf_t *bp, *n;
1850 struct list_head *dwq = &target->bt_delwrite_queue;
1851 spinlock_t *dwlk = &target->bt_delwrite_lock;
1852 int skipped = 0;
1853 int force;
1854
1855 force = test_and_clear_bit(XBT_FORCE_FLUSH, &target->bt_flags);
1856 INIT_LIST_HEAD(list);
1857 spin_lock(dwlk);
1858 list_for_each_entry_safe(bp, n, dwq, b_list) {
1859 ASSERT(bp->b_flags & XBF_DELWRI);
1860
1861 if (!XFS_BUF_ISPINNED(bp) && !xfs_buf_cond_lock(bp)) {
1862 if (!force &&
1863 time_before(jiffies, bp->b_queuetime + age)) {
1864 xfs_buf_unlock(bp);
1865 break;
1866 }
1867
1868 bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q|
1869 _XBF_RUN_QUEUES);
1870 bp->b_flags |= XBF_WRITE;
1871 list_move_tail(&bp->b_list, list);
1872 trace_xfs_buf_delwri_split(bp, _RET_IP_);
1873 } else
1874 skipped++;
1875 }
1876 spin_unlock(dwlk);
1877
1878 return skipped;
1879
1880}
1881
1882
1883
1884
1885
1886
1887static int
1888xfs_buf_cmp(
1889 void *priv,
1890 struct list_head *a,
1891 struct list_head *b)
1892{
1893 struct xfs_buf *ap = container_of(a, struct xfs_buf, b_list);
1894 struct xfs_buf *bp = container_of(b, struct xfs_buf, b_list);
1895 xfs_daddr_t diff;
1896
1897 diff = ap->b_bn - bp->b_bn;
1898 if (diff < 0)
1899 return -1;
1900 if (diff > 0)
1901 return 1;
1902 return 0;
1903}
1904
1905void
1906xfs_buf_delwri_sort(
1907 xfs_buftarg_t *target,
1908 struct list_head *list)
1909{
1910 list_sort(NULL, list, xfs_buf_cmp);
1911}
1912
1913STATIC int
1914xfsbufd(
1915 void *data)
1916{
1917 xfs_buftarg_t *target = (xfs_buftarg_t *)data;
1918
1919 current->flags |= PF_MEMALLOC;
1920
1921 set_freezable();
1922
1923 do {
1924 long age = xfs_buf_age_centisecs * msecs_to_jiffies(10);
1925 long tout = xfs_buf_timer_centisecs * msecs_to_jiffies(10);
1926 int count = 0;
1927 struct list_head tmp;
1928
1929 if (unlikely(freezing(current))) {
1930 set_bit(XBT_FORCE_SLEEP, &target->bt_flags);
1931 refrigerator();
1932 } else {
1933 clear_bit(XBT_FORCE_SLEEP, &target->bt_flags);
1934 }
1935
1936
1937 if (list_empty(&target->bt_delwrite_queue))
1938 tout = MAX_SCHEDULE_TIMEOUT;
1939 schedule_timeout_interruptible(tout);
1940
1941 xfs_buf_delwri_split(target, &tmp, age);
1942 list_sort(NULL, &tmp, xfs_buf_cmp);
1943 while (!list_empty(&tmp)) {
1944 struct xfs_buf *bp;
1945 bp = list_first_entry(&tmp, struct xfs_buf, b_list);
1946 list_del_init(&bp->b_list);
1947 xfs_bdstrat_cb(bp);
1948 count++;
1949 }
1950 if (count)
1951 blk_run_address_space(target->bt_mapping);
1952
1953 } while (!kthread_should_stop());
1954
1955 return 0;
1956}
1957
1958
1959
1960
1961
1962
1963int
1964xfs_flush_buftarg(
1965 xfs_buftarg_t *target,
1966 int wait)
1967{
1968 xfs_buf_t *bp;
1969 int pincount = 0;
1970 LIST_HEAD(tmp_list);
1971 LIST_HEAD(wait_list);
1972
1973 xfs_buf_runall_queues(xfsconvertd_workqueue);
1974 xfs_buf_runall_queues(xfsdatad_workqueue);
1975 xfs_buf_runall_queues(xfslogd_workqueue);
1976
1977 set_bit(XBT_FORCE_FLUSH, &target->bt_flags);
1978 pincount = xfs_buf_delwri_split(target, &tmp_list, 0);
1979
1980
1981
1982
1983
1984
1985 list_sort(NULL, &tmp_list, xfs_buf_cmp);
1986 while (!list_empty(&tmp_list)) {
1987 bp = list_first_entry(&tmp_list, struct xfs_buf, b_list);
1988 ASSERT(target == bp->b_target);
1989 list_del_init(&bp->b_list);
1990 if (wait) {
1991 bp->b_flags &= ~XBF_ASYNC;
1992 list_add(&bp->b_list, &wait_list);
1993 }
1994 xfs_bdstrat_cb(bp);
1995 }
1996
1997 if (wait) {
1998
1999 blk_run_address_space(target->bt_mapping);
2000 while (!list_empty(&wait_list)) {
2001 bp = list_first_entry(&wait_list, struct xfs_buf, b_list);
2002
2003 list_del_init(&bp->b_list);
2004 xfs_buf_iowait(bp);
2005 xfs_buf_relse(bp);
2006 }
2007 }
2008
2009 return pincount;
2010}
2011
2012int __init
2013xfs_buf_init(void)
2014{
2015 xfs_buf_zone = kmem_zone_init_flags(sizeof(xfs_buf_t), "xfs_buf",
2016 KM_ZONE_HWALIGN, NULL);
2017 if (!xfs_buf_zone)
2018 goto out;
2019
2020 xfslogd_workqueue = alloc_workqueue("xfslogd",
2021 WQ_MEM_RECLAIM | WQ_HIGHPRI, 1);
2022 if (!xfslogd_workqueue)
2023 goto out_free_buf_zone;
2024
2025 xfsdatad_workqueue = create_workqueue("xfsdatad");
2026 if (!xfsdatad_workqueue)
2027 goto out_destroy_xfslogd_workqueue;
2028
2029 xfsconvertd_workqueue = create_workqueue("xfsconvertd");
2030 if (!xfsconvertd_workqueue)
2031 goto out_destroy_xfsdatad_workqueue;
2032
2033 return 0;
2034
2035 out_destroy_xfsdatad_workqueue:
2036 destroy_workqueue(xfsdatad_workqueue);
2037 out_destroy_xfslogd_workqueue:
2038 destroy_workqueue(xfslogd_workqueue);
2039 out_free_buf_zone:
2040 kmem_zone_destroy(xfs_buf_zone);
2041 out:
2042 return -ENOMEM;
2043}
2044
2045void
2046xfs_buf_terminate(void)
2047{
2048 destroy_workqueue(xfsconvertd_workqueue);
2049 destroy_workqueue(xfsdatad_workqueue);
2050 destroy_workqueue(xfslogd_workqueue);
2051 kmem_zone_destroy(xfs_buf_zone);
2052}
2053
2054#ifdef CONFIG_KDB_MODULES
2055struct list_head *
2056xfs_get_buftarg_list(void)
2057{
2058 return &xfs_buftarg_list;
2059}
2060#endif
2061