1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18#include "xfs.h"
19#include <linux/stddef.h>
20#include <linux/errno.h>
21#include <linux/gfp.h>
22#include <linux/pagemap.h>
23#include <linux/init.h>
24#include <linux/vmalloc.h>
25#include <linux/bio.h>
26#include <linux/sysctl.h>
27#include <linux/proc_fs.h>
28#include <linux/workqueue.h>
29#include <linux/percpu.h>
30#include <linux/blkdev.h>
31#include <linux/hash.h>
32#include <linux/kthread.h>
33#include <linux/migrate.h>
34#include <linux/backing-dev.h>
35#include <linux/freezer.h>
36
37#include "xfs_sb.h"
38#include "xfs_log.h"
39#include "xfs_ag.h"
40#include "xfs_mount.h"
41#include "xfs_trace.h"
42
43static kmem_zone_t *xfs_buf_zone;
44
45static struct workqueue_struct *xfslogd_workqueue;
46
47#ifdef XFS_BUF_LOCK_TRACKING
48# define XB_SET_OWNER(bp) ((bp)->b_last_holder = current->pid)
49# define XB_CLEAR_OWNER(bp) ((bp)->b_last_holder = -1)
50# define XB_GET_OWNER(bp) ((bp)->b_last_holder)
51#else
52# define XB_SET_OWNER(bp) do { } while (0)
53# define XB_CLEAR_OWNER(bp) do { } while (0)
54# define XB_GET_OWNER(bp) do { } while (0)
55#endif
56
57#define xb_to_gfp(flags) \
58 ((((flags) & XBF_READ_AHEAD) ? __GFP_NORETRY : GFP_NOFS) | __GFP_NOWARN)
59
60
61static inline int
62xfs_buf_is_vmapped(
63 struct xfs_buf *bp)
64{
65
66
67
68
69
70
71
72 return bp->b_addr && bp->b_page_count > 1;
73}
74
75static inline int
76xfs_buf_vmap_len(
77 struct xfs_buf *bp)
78{
79 return (bp->b_page_count * PAGE_SIZE) - bp->b_offset;
80}
81
82
83
84
85
86
87
88STATIC void
89xfs_buf_lru_add(
90 struct xfs_buf *bp)
91{
92 struct xfs_buftarg *btp = bp->b_target;
93
94 spin_lock(&btp->bt_lru_lock);
95 if (list_empty(&bp->b_lru)) {
96 atomic_inc(&bp->b_hold);
97 list_add_tail(&bp->b_lru, &btp->bt_lru);
98 btp->bt_lru_nr++;
99 bp->b_lru_flags &= ~_XBF_LRU_DISPOSE;
100 }
101 spin_unlock(&btp->bt_lru_lock);
102}
103
104
105
106
107
108
109
110
111
112
113STATIC void
114xfs_buf_lru_del(
115 struct xfs_buf *bp)
116{
117 struct xfs_buftarg *btp = bp->b_target;
118
119 if (list_empty(&bp->b_lru))
120 return;
121
122 spin_lock(&btp->bt_lru_lock);
123 if (!list_empty(&bp->b_lru)) {
124 list_del_init(&bp->b_lru);
125 btp->bt_lru_nr--;
126 }
127 spin_unlock(&btp->bt_lru_lock);
128}
129
130
131
132
133
134
135
136
137
138void
139xfs_buf_stale(
140 struct xfs_buf *bp)
141{
142 ASSERT(xfs_buf_islocked(bp));
143
144 bp->b_flags |= XBF_STALE;
145
146
147
148
149
150
151 bp->b_flags &= ~_XBF_DELWRI_Q;
152
153 atomic_set(&(bp)->b_lru_ref, 0);
154 if (!list_empty(&bp->b_lru)) {
155 struct xfs_buftarg *btp = bp->b_target;
156
157 spin_lock(&btp->bt_lru_lock);
158 if (!list_empty(&bp->b_lru) &&
159 !(bp->b_lru_flags & _XBF_LRU_DISPOSE)) {
160 list_del_init(&bp->b_lru);
161 btp->bt_lru_nr--;
162 atomic_dec(&bp->b_hold);
163 }
164 spin_unlock(&btp->bt_lru_lock);
165 }
166 ASSERT(atomic_read(&bp->b_hold) >= 1);
167}
168
169static int
170xfs_buf_get_maps(
171 struct xfs_buf *bp,
172 int map_count)
173{
174 ASSERT(bp->b_maps == NULL);
175 bp->b_map_count = map_count;
176
177 if (map_count == 1) {
178 bp->b_maps = &bp->__b_map;
179 return 0;
180 }
181
182 bp->b_maps = kmem_zalloc(map_count * sizeof(struct xfs_buf_map),
183 KM_NOFS);
184 if (!bp->b_maps)
185 return ENOMEM;
186 return 0;
187}
188
189
190
191
192static void
193xfs_buf_free_maps(
194 struct xfs_buf *bp)
195{
196 if (bp->b_maps != &bp->__b_map) {
197 kmem_free(bp->b_maps);
198 bp->b_maps = NULL;
199 }
200}
201
202struct xfs_buf *
203_xfs_buf_alloc(
204 struct xfs_buftarg *target,
205 struct xfs_buf_map *map,
206 int nmaps,
207 xfs_buf_flags_t flags)
208{
209 struct xfs_buf *bp;
210 int error;
211 int i;
212
213 bp = kmem_zone_zalloc(xfs_buf_zone, KM_NOFS);
214 if (unlikely(!bp))
215 return NULL;
216
217
218
219
220
221 flags &= ~(XBF_UNMAPPED | XBF_TRYLOCK | XBF_ASYNC | XBF_READ_AHEAD);
222
223 atomic_set(&bp->b_hold, 1);
224 atomic_set(&bp->b_lru_ref, 1);
225 init_completion(&bp->b_iowait);
226 INIT_LIST_HEAD(&bp->b_lru);
227 INIT_LIST_HEAD(&bp->b_list);
228 RB_CLEAR_NODE(&bp->b_rbnode);
229 sema_init(&bp->b_sema, 0);
230 XB_SET_OWNER(bp);
231 bp->b_target = target;
232 bp->b_flags = flags;
233
234
235
236
237
238
239 error = xfs_buf_get_maps(bp, nmaps);
240 if (error) {
241 kmem_zone_free(xfs_buf_zone, bp);
242 return NULL;
243 }
244
245 bp->b_bn = map[0].bm_bn;
246 bp->b_length = 0;
247 for (i = 0; i < nmaps; i++) {
248 bp->b_maps[i].bm_bn = map[i].bm_bn;
249 bp->b_maps[i].bm_len = map[i].bm_len;
250 bp->b_length += map[i].bm_len;
251 }
252 bp->b_io_length = bp->b_length;
253
254 atomic_set(&bp->b_pin_count, 0);
255 init_waitqueue_head(&bp->b_waiters);
256
257 XFS_STATS_INC(xb_create);
258 trace_xfs_buf_init(bp, _RET_IP_);
259
260 return bp;
261}
262
263
264
265
266
267STATIC int
268_xfs_buf_get_pages(
269 xfs_buf_t *bp,
270 int page_count,
271 xfs_buf_flags_t flags)
272{
273
274 if (bp->b_pages == NULL) {
275 bp->b_page_count = page_count;
276 if (page_count <= XB_PAGES) {
277 bp->b_pages = bp->b_page_array;
278 } else {
279 bp->b_pages = kmem_alloc(sizeof(struct page *) *
280 page_count, KM_NOFS);
281 if (bp->b_pages == NULL)
282 return -ENOMEM;
283 }
284 memset(bp->b_pages, 0, sizeof(struct page *) * page_count);
285 }
286 return 0;
287}
288
289
290
291
292STATIC void
293_xfs_buf_free_pages(
294 xfs_buf_t *bp)
295{
296 if (bp->b_pages != bp->b_page_array) {
297 kmem_free(bp->b_pages);
298 bp->b_pages = NULL;
299 }
300}
301
302
303
304
305
306
307
308
309void
310xfs_buf_free(
311 xfs_buf_t *bp)
312{
313 trace_xfs_buf_free(bp, _RET_IP_);
314
315 ASSERT(list_empty(&bp->b_lru));
316
317 if (bp->b_flags & _XBF_PAGES) {
318 uint i;
319
320 if (xfs_buf_is_vmapped(bp))
321 vm_unmap_ram(bp->b_addr - bp->b_offset,
322 bp->b_page_count);
323
324 for (i = 0; i < bp->b_page_count; i++) {
325 struct page *page = bp->b_pages[i];
326
327 __free_page(page);
328 }
329 } else if (bp->b_flags & _XBF_KMEM)
330 kmem_free(bp->b_addr);
331 _xfs_buf_free_pages(bp);
332 xfs_buf_free_maps(bp);
333 kmem_zone_free(xfs_buf_zone, bp);
334}
335
336
337
338
339STATIC int
340xfs_buf_allocate_memory(
341 xfs_buf_t *bp,
342 uint flags)
343{
344 size_t size;
345 size_t nbytes, offset;
346 gfp_t gfp_mask = xb_to_gfp(flags);
347 unsigned short page_count, i;
348 xfs_off_t start, end;
349 int error;
350
351
352
353
354
355
356 size = BBTOB(bp->b_length);
357 if (size < PAGE_SIZE) {
358 bp->b_addr = kmem_alloc(size, KM_NOFS);
359 if (!bp->b_addr) {
360
361 goto use_alloc_page;
362 }
363
364 if (((unsigned long)(bp->b_addr + size - 1) & PAGE_MASK) !=
365 ((unsigned long)bp->b_addr & PAGE_MASK)) {
366
367 kmem_free(bp->b_addr);
368 bp->b_addr = NULL;
369 goto use_alloc_page;
370 }
371 bp->b_offset = offset_in_page(bp->b_addr);
372 bp->b_pages = bp->b_page_array;
373 bp->b_pages[0] = virt_to_page(bp->b_addr);
374 bp->b_page_count = 1;
375 bp->b_flags |= _XBF_KMEM;
376 return 0;
377 }
378
379use_alloc_page:
380 start = BBTOB(bp->b_maps[0].bm_bn) >> PAGE_SHIFT;
381 end = (BBTOB(bp->b_maps[0].bm_bn + bp->b_length) + PAGE_SIZE - 1)
382 >> PAGE_SHIFT;
383 page_count = end - start;
384 error = _xfs_buf_get_pages(bp, page_count, flags);
385 if (unlikely(error))
386 return error;
387
388 offset = bp->b_offset;
389 bp->b_flags |= _XBF_PAGES;
390
391 for (i = 0; i < bp->b_page_count; i++) {
392 struct page *page;
393 uint retries = 0;
394retry:
395 page = alloc_page(gfp_mask);
396 if (unlikely(page == NULL)) {
397 if (flags & XBF_READ_AHEAD) {
398 bp->b_page_count = i;
399 error = ENOMEM;
400 goto out_free_pages;
401 }
402
403
404
405
406
407
408
409 if (!(++retries % 100))
410 xfs_err(NULL,
411 "possible memory allocation deadlock in %s (mode:0x%x)",
412 __func__, gfp_mask);
413
414 XFS_STATS_INC(xb_page_retries);
415 congestion_wait(BLK_RW_ASYNC, HZ/50);
416 goto retry;
417 }
418
419 XFS_STATS_INC(xb_page_found);
420
421 nbytes = min_t(size_t, size, PAGE_SIZE - offset);
422 size -= nbytes;
423 bp->b_pages[i] = page;
424 offset = 0;
425 }
426 return 0;
427
428out_free_pages:
429 for (i = 0; i < bp->b_page_count; i++)
430 __free_page(bp->b_pages[i]);
431 return error;
432}
433
434
435
436
437STATIC int
438_xfs_buf_map_pages(
439 xfs_buf_t *bp,
440 uint flags)
441{
442 ASSERT(bp->b_flags & _XBF_PAGES);
443 if (bp->b_page_count == 1) {
444
445 bp->b_addr = page_address(bp->b_pages[0]) + bp->b_offset;
446 } else if (flags & XBF_UNMAPPED) {
447 bp->b_addr = NULL;
448 } else {
449 int retried = 0;
450
451 do {
452 bp->b_addr = vm_map_ram(bp->b_pages, bp->b_page_count,
453 -1, PAGE_KERNEL);
454 if (bp->b_addr)
455 break;
456 vm_unmap_aliases();
457 } while (retried++ <= 1);
458
459 if (!bp->b_addr)
460 return -ENOMEM;
461 bp->b_addr += bp->b_offset;
462 }
463
464 return 0;
465}
466
467
468
469
470
471
472
473
474
475
476xfs_buf_t *
477_xfs_buf_find(
478 struct xfs_buftarg *btp,
479 struct xfs_buf_map *map,
480 int nmaps,
481 xfs_buf_flags_t flags,
482 xfs_buf_t *new_bp)
483{
484 size_t numbytes;
485 struct xfs_perag *pag;
486 struct rb_node **rbp;
487 struct rb_node *parent;
488 xfs_buf_t *bp;
489 xfs_daddr_t blkno = map[0].bm_bn;
490 xfs_daddr_t eofs;
491 int numblks = 0;
492 int i;
493
494 for (i = 0; i < nmaps; i++)
495 numblks += map[i].bm_len;
496 numbytes = BBTOB(numblks);
497
498
499 ASSERT(!(numbytes < (1 << btp->bt_sshift)));
500 ASSERT(!(BBTOB(blkno) & (xfs_off_t)btp->bt_smask));
501
502
503
504
505
506 eofs = XFS_FSB_TO_BB(btp->bt_mount, btp->bt_mount->m_sb.sb_dblocks);
507 if (blkno >= eofs) {
508
509
510
511
512
513 xfs_alert(btp->bt_mount,
514 "%s: Block out of range: block 0x%llx, EOFS 0x%llx ",
515 __func__, blkno, eofs);
516 WARN_ON(1);
517 return NULL;
518 }
519
520
521 pag = xfs_perag_get(btp->bt_mount,
522 xfs_daddr_to_agno(btp->bt_mount, blkno));
523
524
525 spin_lock(&pag->pag_buf_lock);
526 rbp = &pag->pag_buf_tree.rb_node;
527 parent = NULL;
528 bp = NULL;
529 while (*rbp) {
530 parent = *rbp;
531 bp = rb_entry(parent, struct xfs_buf, b_rbnode);
532
533 if (blkno < bp->b_bn)
534 rbp = &(*rbp)->rb_left;
535 else if (blkno > bp->b_bn)
536 rbp = &(*rbp)->rb_right;
537 else {
538
539
540
541
542
543
544
545
546 if (bp->b_length != numblks) {
547 ASSERT(bp->b_flags & XBF_STALE);
548 rbp = &(*rbp)->rb_right;
549 continue;
550 }
551 atomic_inc(&bp->b_hold);
552 goto found;
553 }
554 }
555
556
557 if (new_bp) {
558 rb_link_node(&new_bp->b_rbnode, parent, rbp);
559 rb_insert_color(&new_bp->b_rbnode, &pag->pag_buf_tree);
560
561 new_bp->b_pag = pag;
562 spin_unlock(&pag->pag_buf_lock);
563 } else {
564 XFS_STATS_INC(xb_miss_locked);
565 spin_unlock(&pag->pag_buf_lock);
566 xfs_perag_put(pag);
567 }
568 return new_bp;
569
570found:
571 spin_unlock(&pag->pag_buf_lock);
572 xfs_perag_put(pag);
573
574 if (!xfs_buf_trylock(bp)) {
575 if (flags & XBF_TRYLOCK) {
576 xfs_buf_rele(bp);
577 XFS_STATS_INC(xb_busy_locked);
578 return NULL;
579 }
580 xfs_buf_lock(bp);
581 XFS_STATS_INC(xb_get_locked_waited);
582 }
583
584
585
586
587
588
589 if (bp->b_flags & XBF_STALE) {
590 ASSERT((bp->b_flags & _XBF_DELWRI_Q) == 0);
591 ASSERT(bp->b_iodone == NULL);
592 bp->b_flags &= _XBF_KMEM | _XBF_PAGES;
593 bp->b_ops = NULL;
594 }
595
596 trace_xfs_buf_find(bp, flags, _RET_IP_);
597 XFS_STATS_INC(xb_get_locked);
598 return bp;
599}
600
601
602
603
604
605
606struct xfs_buf *
607xfs_buf_get_map(
608 struct xfs_buftarg *target,
609 struct xfs_buf_map *map,
610 int nmaps,
611 xfs_buf_flags_t flags)
612{
613 struct xfs_buf *bp;
614 struct xfs_buf *new_bp;
615 int error = 0;
616
617 bp = _xfs_buf_find(target, map, nmaps, flags, NULL);
618 if (likely(bp))
619 goto found;
620
621 new_bp = _xfs_buf_alloc(target, map, nmaps, flags);
622 if (unlikely(!new_bp))
623 return NULL;
624
625 error = xfs_buf_allocate_memory(new_bp, flags);
626 if (error) {
627 xfs_buf_free(new_bp);
628 return NULL;
629 }
630
631 bp = _xfs_buf_find(target, map, nmaps, flags, new_bp);
632 if (!bp) {
633 xfs_buf_free(new_bp);
634 return NULL;
635 }
636
637 if (bp != new_bp)
638 xfs_buf_free(new_bp);
639
640found:
641 if (!bp->b_addr) {
642 error = _xfs_buf_map_pages(bp, flags);
643 if (unlikely(error)) {
644 xfs_warn(target->bt_mount,
645 "%s: failed to map pages\n", __func__);
646 xfs_buf_relse(bp);
647 return NULL;
648 }
649 }
650
651 XFS_STATS_INC(xb_get);
652 trace_xfs_buf_get(bp, flags, _RET_IP_);
653 return bp;
654}
655
656STATIC int
657_xfs_buf_read(
658 xfs_buf_t *bp,
659 xfs_buf_flags_t flags)
660{
661 ASSERT(!(flags & XBF_WRITE));
662 ASSERT(bp->b_maps[0].bm_bn != XFS_BUF_DADDR_NULL);
663
664 bp->b_flags &= ~(XBF_WRITE | XBF_ASYNC | XBF_READ_AHEAD);
665 bp->b_flags |= flags & (XBF_READ | XBF_ASYNC | XBF_READ_AHEAD);
666
667 xfs_buf_iorequest(bp);
668 if (flags & XBF_ASYNC)
669 return 0;
670 return xfs_buf_iowait(bp);
671}
672
673xfs_buf_t *
674xfs_buf_read_map(
675 struct xfs_buftarg *target,
676 struct xfs_buf_map *map,
677 int nmaps,
678 xfs_buf_flags_t flags,
679 const struct xfs_buf_ops *ops)
680{
681 struct xfs_buf *bp;
682
683 flags |= XBF_READ;
684
685 bp = xfs_buf_get_map(target, map, nmaps, flags);
686 if (bp) {
687 trace_xfs_buf_read(bp, flags, _RET_IP_);
688
689 if (!XFS_BUF_ISDONE(bp)) {
690 XFS_STATS_INC(xb_get_read);
691 bp->b_ops = ops;
692 _xfs_buf_read(bp, flags);
693 } else if (flags & XBF_ASYNC) {
694
695
696
697
698 xfs_buf_relse(bp);
699 return NULL;
700 } else {
701
702 bp->b_flags &= ~XBF_READ;
703 }
704 }
705
706 return bp;
707}
708
709
710
711
712
713void
714xfs_buf_readahead_map(
715 struct xfs_buftarg *target,
716 struct xfs_buf_map *map,
717 int nmaps,
718 const struct xfs_buf_ops *ops)
719{
720 if (bdi_read_congested(target->bt_bdi))
721 return;
722
723 xfs_buf_read_map(target, map, nmaps,
724 XBF_TRYLOCK|XBF_ASYNC|XBF_READ_AHEAD, ops);
725}
726
727
728
729
730
731struct xfs_buf *
732xfs_buf_read_uncached(
733 struct xfs_buftarg *target,
734 xfs_daddr_t daddr,
735 size_t numblks,
736 int flags,
737 const struct xfs_buf_ops *ops)
738{
739 struct xfs_buf *bp;
740
741 bp = xfs_buf_get_uncached(target, numblks, flags);
742 if (!bp)
743 return NULL;
744
745
746 ASSERT(bp->b_map_count == 1);
747 bp->b_bn = daddr;
748 bp->b_maps[0].bm_bn = daddr;
749 bp->b_flags |= XBF_READ;
750 bp->b_ops = ops;
751
752 xfsbdstrat(target->bt_mount, bp);
753 xfs_buf_iowait(bp);
754 return bp;
755}
756
757
758
759
760
761void
762xfs_buf_set_empty(
763 struct xfs_buf *bp,
764 size_t numblks)
765{
766 if (bp->b_pages)
767 _xfs_buf_free_pages(bp);
768
769 bp->b_pages = NULL;
770 bp->b_page_count = 0;
771 bp->b_addr = NULL;
772 bp->b_length = numblks;
773 bp->b_io_length = numblks;
774
775 ASSERT(bp->b_map_count == 1);
776 bp->b_bn = XFS_BUF_DADDR_NULL;
777 bp->b_maps[0].bm_bn = XFS_BUF_DADDR_NULL;
778 bp->b_maps[0].bm_len = bp->b_length;
779}
780
781static inline struct page *
782mem_to_page(
783 void *addr)
784{
785 if ((!is_vmalloc_addr(addr))) {
786 return virt_to_page(addr);
787 } else {
788 return vmalloc_to_page(addr);
789 }
790}
791
792int
793xfs_buf_associate_memory(
794 xfs_buf_t *bp,
795 void *mem,
796 size_t len)
797{
798 int rval;
799 int i = 0;
800 unsigned long pageaddr;
801 unsigned long offset;
802 size_t buflen;
803 int page_count;
804
805 pageaddr = (unsigned long)mem & PAGE_MASK;
806 offset = (unsigned long)mem - pageaddr;
807 buflen = PAGE_ALIGN(len + offset);
808 page_count = buflen >> PAGE_SHIFT;
809
810
811 if (bp->b_pages)
812 _xfs_buf_free_pages(bp);
813
814 bp->b_pages = NULL;
815 bp->b_addr = mem;
816
817 rval = _xfs_buf_get_pages(bp, page_count, 0);
818 if (rval)
819 return rval;
820
821 bp->b_offset = offset;
822
823 for (i = 0; i < bp->b_page_count; i++) {
824 bp->b_pages[i] = mem_to_page((void *)pageaddr);
825 pageaddr += PAGE_SIZE;
826 }
827
828 bp->b_io_length = BTOBB(len);
829 bp->b_length = BTOBB(buflen);
830
831 return 0;
832}
833
834xfs_buf_t *
835xfs_buf_get_uncached(
836 struct xfs_buftarg *target,
837 size_t numblks,
838 int flags)
839{
840 unsigned long page_count;
841 int error, i;
842 struct xfs_buf *bp;
843 DEFINE_SINGLE_BUF_MAP(map, XFS_BUF_DADDR_NULL, numblks);
844
845 bp = _xfs_buf_alloc(target, &map, 1, 0);
846 if (unlikely(bp == NULL))
847 goto fail;
848
849 page_count = PAGE_ALIGN(numblks << BBSHIFT) >> PAGE_SHIFT;
850 error = _xfs_buf_get_pages(bp, page_count, 0);
851 if (error)
852 goto fail_free_buf;
853
854 for (i = 0; i < page_count; i++) {
855 bp->b_pages[i] = alloc_page(xb_to_gfp(flags));
856 if (!bp->b_pages[i])
857 goto fail_free_mem;
858 }
859 bp->b_flags |= _XBF_PAGES;
860
861 error = _xfs_buf_map_pages(bp, 0);
862 if (unlikely(error)) {
863 xfs_warn(target->bt_mount,
864 "%s: failed to map pages\n", __func__);
865 goto fail_free_mem;
866 }
867
868 trace_xfs_buf_get_uncached(bp, _RET_IP_);
869 return bp;
870
871 fail_free_mem:
872 while (--i >= 0)
873 __free_page(bp->b_pages[i]);
874 _xfs_buf_free_pages(bp);
875 fail_free_buf:
876 xfs_buf_free_maps(bp);
877 kmem_zone_free(xfs_buf_zone, bp);
878 fail:
879 return NULL;
880}
881
882
883
884
885
886
887void
888xfs_buf_hold(
889 xfs_buf_t *bp)
890{
891 trace_xfs_buf_hold(bp, _RET_IP_);
892 atomic_inc(&bp->b_hold);
893}
894
895
896
897
898
899void
900xfs_buf_rele(
901 xfs_buf_t *bp)
902{
903 struct xfs_perag *pag = bp->b_pag;
904
905 trace_xfs_buf_rele(bp, _RET_IP_);
906
907 if (!pag) {
908 ASSERT(list_empty(&bp->b_lru));
909 ASSERT(RB_EMPTY_NODE(&bp->b_rbnode));
910 if (atomic_dec_and_test(&bp->b_hold))
911 xfs_buf_free(bp);
912 return;
913 }
914
915 ASSERT(!RB_EMPTY_NODE(&bp->b_rbnode));
916
917 ASSERT(atomic_read(&bp->b_hold) > 0);
918 if (atomic_dec_and_lock(&bp->b_hold, &pag->pag_buf_lock)) {
919 if (!(bp->b_flags & XBF_STALE) &&
920 atomic_read(&bp->b_lru_ref)) {
921 xfs_buf_lru_add(bp);
922 spin_unlock(&pag->pag_buf_lock);
923 } else {
924 xfs_buf_lru_del(bp);
925 ASSERT(!(bp->b_flags & _XBF_DELWRI_Q));
926 rb_erase(&bp->b_rbnode, &pag->pag_buf_tree);
927 spin_unlock(&pag->pag_buf_lock);
928 xfs_perag_put(pag);
929 xfs_buf_free(bp);
930 }
931 }
932}
933
934
935
936
937
938
939
940
941
942
943
944
945
946int
947xfs_buf_trylock(
948 struct xfs_buf *bp)
949{
950 int locked;
951
952 locked = down_trylock(&bp->b_sema) == 0;
953 if (locked)
954 XB_SET_OWNER(bp);
955
956 trace_xfs_buf_trylock(bp, _RET_IP_);
957 return locked;
958}
959
960
961
962
963
964
965
966
967
968
969void
970xfs_buf_lock(
971 struct xfs_buf *bp)
972{
973 trace_xfs_buf_lock(bp, _RET_IP_);
974
975 if (atomic_read(&bp->b_pin_count) && (bp->b_flags & XBF_STALE))
976 xfs_log_force(bp->b_target->bt_mount, 0);
977 down(&bp->b_sema);
978 XB_SET_OWNER(bp);
979
980 trace_xfs_buf_lock_done(bp, _RET_IP_);
981}
982
983void
984xfs_buf_unlock(
985 struct xfs_buf *bp)
986{
987 XB_CLEAR_OWNER(bp);
988 up(&bp->b_sema);
989
990 trace_xfs_buf_unlock(bp, _RET_IP_);
991}
992
993STATIC void
994xfs_buf_wait_unpin(
995 xfs_buf_t *bp)
996{
997 DECLARE_WAITQUEUE (wait, current);
998
999 if (atomic_read(&bp->b_pin_count) == 0)
1000 return;
1001
1002 add_wait_queue(&bp->b_waiters, &wait);
1003 for (;;) {
1004 set_current_state(TASK_UNINTERRUPTIBLE);
1005 if (atomic_read(&bp->b_pin_count) == 0)
1006 break;
1007 io_schedule();
1008 }
1009 remove_wait_queue(&bp->b_waiters, &wait);
1010 set_current_state(TASK_RUNNING);
1011}
1012
1013
1014
1015
1016
1017STATIC void
1018xfs_buf_iodone_work(
1019 struct work_struct *work)
1020{
1021 struct xfs_buf *bp =
1022 container_of(work, xfs_buf_t, b_iodone_work);
1023 bool read = !!(bp->b_flags & XBF_READ);
1024
1025 bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_READ_AHEAD);
1026
1027
1028 if (read && bp->b_ops && !bp->b_error && (bp->b_flags & XBF_DONE))
1029 bp->b_ops->verify_read(bp);
1030
1031 if (bp->b_iodone)
1032 (*(bp->b_iodone))(bp);
1033 else if (bp->b_flags & XBF_ASYNC)
1034 xfs_buf_relse(bp);
1035 else {
1036 ASSERT(read && bp->b_ops);
1037 complete(&bp->b_iowait);
1038 }
1039}
1040
1041void
1042xfs_buf_ioend(
1043 struct xfs_buf *bp,
1044 int schedule)
1045{
1046 bool read = !!(bp->b_flags & XBF_READ);
1047
1048 trace_xfs_buf_iodone(bp, _RET_IP_);
1049
1050 if (bp->b_error == 0)
1051 bp->b_flags |= XBF_DONE;
1052
1053 if (bp->b_iodone || (read && bp->b_ops) || (bp->b_flags & XBF_ASYNC)) {
1054 if (schedule) {
1055 INIT_WORK(&bp->b_iodone_work, xfs_buf_iodone_work);
1056 queue_work(xfslogd_workqueue, &bp->b_iodone_work);
1057 } else {
1058 xfs_buf_iodone_work(&bp->b_iodone_work);
1059 }
1060 } else {
1061 bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_READ_AHEAD);
1062 complete(&bp->b_iowait);
1063 }
1064}
1065
1066void
1067xfs_buf_ioerror(
1068 xfs_buf_t *bp,
1069 int error)
1070{
1071 ASSERT(error >= 0 && error <= 0xffff);
1072 bp->b_error = (unsigned short)error;
1073 trace_xfs_buf_ioerror(bp, error, _RET_IP_);
1074}
1075
1076void
1077xfs_buf_ioerror_alert(
1078 struct xfs_buf *bp,
1079 const char *func)
1080{
1081 xfs_alert(bp->b_target->bt_mount,
1082"metadata I/O error: block 0x%llx (\"%s\") error %d numblks %d",
1083 (__uint64_t)XFS_BUF_ADDR(bp), func, bp->b_error, bp->b_length);
1084}
1085
1086
1087
1088
1089
1090
1091STATIC int
1092xfs_bioerror(
1093 xfs_buf_t *bp)
1094{
1095#ifdef XFSERRORDEBUG
1096 ASSERT(XFS_BUF_ISREAD(bp) || bp->b_iodone);
1097#endif
1098
1099
1100
1101
1102 xfs_buf_ioerror(bp, EIO);
1103
1104
1105
1106
1107 XFS_BUF_UNREAD(bp);
1108 XFS_BUF_UNDONE(bp);
1109 xfs_buf_stale(bp);
1110
1111 xfs_buf_ioend(bp, 0);
1112
1113 return EIO;
1114}
1115
1116
1117
1118
1119
1120
1121
1122STATIC int
1123xfs_bioerror_relse(
1124 struct xfs_buf *bp)
1125{
1126 int64_t fl = bp->b_flags;
1127
1128
1129
1130
1131
1132
1133
1134
1135 XFS_BUF_UNREAD(bp);
1136 XFS_BUF_DONE(bp);
1137 xfs_buf_stale(bp);
1138 bp->b_iodone = NULL;
1139 if (!(fl & XBF_ASYNC)) {
1140
1141
1142
1143
1144
1145
1146 xfs_buf_ioerror(bp, EIO);
1147 complete(&bp->b_iowait);
1148 } else {
1149 xfs_buf_relse(bp);
1150 }
1151
1152 return EIO;
1153}
1154
1155STATIC int
1156xfs_bdstrat_cb(
1157 struct xfs_buf *bp)
1158{
1159 if (XFS_FORCED_SHUTDOWN(bp->b_target->bt_mount)) {
1160 trace_xfs_bdstrat_shut(bp, _RET_IP_);
1161
1162
1163
1164
1165
1166 if (!bp->b_iodone && !XFS_BUF_ISREAD(bp))
1167 return xfs_bioerror_relse(bp);
1168 else
1169 return xfs_bioerror(bp);
1170 }
1171
1172 xfs_buf_iorequest(bp);
1173 return 0;
1174}
1175
1176int
1177xfs_bwrite(
1178 struct xfs_buf *bp)
1179{
1180 int error;
1181
1182 ASSERT(xfs_buf_islocked(bp));
1183
1184 bp->b_flags |= XBF_WRITE;
1185 bp->b_flags &= ~(XBF_ASYNC | XBF_READ | _XBF_DELWRI_Q);
1186
1187 xfs_bdstrat_cb(bp);
1188
1189 error = xfs_buf_iowait(bp);
1190 if (error) {
1191 xfs_force_shutdown(bp->b_target->bt_mount,
1192 SHUTDOWN_META_IO_ERROR);
1193 }
1194 return error;
1195}
1196
1197
1198
1199
1200
1201
1202void
1203xfsbdstrat(
1204 struct xfs_mount *mp,
1205 struct xfs_buf *bp)
1206{
1207 if (XFS_FORCED_SHUTDOWN(mp)) {
1208 trace_xfs_bdstrat_shut(bp, _RET_IP_);
1209 xfs_bioerror_relse(bp);
1210 return;
1211 }
1212
1213 xfs_buf_iorequest(bp);
1214}
1215
1216STATIC void
1217_xfs_buf_ioend(
1218 xfs_buf_t *bp,
1219 int schedule)
1220{
1221 if (atomic_dec_and_test(&bp->b_io_remaining) == 1)
1222 xfs_buf_ioend(bp, schedule);
1223}
1224
1225STATIC void
1226xfs_buf_bio_end_io(
1227 struct bio *bio,
1228 int error)
1229{
1230 xfs_buf_t *bp = (xfs_buf_t *)bio->bi_private;
1231
1232
1233
1234
1235
1236 if (!bp->b_error)
1237 xfs_buf_ioerror(bp, -error);
1238
1239 if (!bp->b_error && xfs_buf_is_vmapped(bp) && (bp->b_flags & XBF_READ))
1240 invalidate_kernel_vmap_range(bp->b_addr, xfs_buf_vmap_len(bp));
1241
1242 _xfs_buf_ioend(bp, 1);
1243 bio_put(bio);
1244}
1245
1246static void
1247xfs_buf_ioapply_map(
1248 struct xfs_buf *bp,
1249 int map,
1250 int *buf_offset,
1251 int *count,
1252 int rw)
1253{
1254 int page_index;
1255 int total_nr_pages = bp->b_page_count;
1256 int nr_pages;
1257 struct bio *bio;
1258 sector_t sector = bp->b_maps[map].bm_bn;
1259 int size;
1260 int offset;
1261
1262 total_nr_pages = bp->b_page_count;
1263
1264
1265 page_index = 0;
1266 offset = *buf_offset;
1267 while (offset >= PAGE_SIZE) {
1268 page_index++;
1269 offset -= PAGE_SIZE;
1270 }
1271
1272
1273
1274
1275
1276 size = min_t(int, BBTOB(bp->b_maps[map].bm_len), *count);
1277 *count -= size;
1278 *buf_offset += size;
1279
1280next_chunk:
1281 atomic_inc(&bp->b_io_remaining);
1282 nr_pages = BIO_MAX_SECTORS >> (PAGE_SHIFT - BBSHIFT);
1283 if (nr_pages > total_nr_pages)
1284 nr_pages = total_nr_pages;
1285
1286 bio = bio_alloc(GFP_NOIO, nr_pages);
1287 bio->bi_bdev = bp->b_target->bt_bdev;
1288 bio->bi_sector = sector;
1289 bio->bi_end_io = xfs_buf_bio_end_io;
1290 bio->bi_private = bp;
1291
1292
1293 for (; size && nr_pages; nr_pages--, page_index++) {
1294 int rbytes, nbytes = PAGE_SIZE - offset;
1295
1296 if (nbytes > size)
1297 nbytes = size;
1298
1299 rbytes = bio_add_page(bio, bp->b_pages[page_index], nbytes,
1300 offset);
1301 if (rbytes < nbytes)
1302 break;
1303
1304 offset = 0;
1305 sector += BTOBB(nbytes);
1306 size -= nbytes;
1307 total_nr_pages--;
1308 }
1309
1310 if (likely(bio->bi_size)) {
1311 if (xfs_buf_is_vmapped(bp)) {
1312 flush_kernel_vmap_range(bp->b_addr,
1313 xfs_buf_vmap_len(bp));
1314 }
1315 submit_bio(rw, bio);
1316 if (size)
1317 goto next_chunk;
1318 } else {
1319
1320
1321
1322
1323 atomic_dec(&bp->b_io_remaining);
1324 xfs_buf_ioerror(bp, EIO);
1325 bio_put(bio);
1326 }
1327
1328}
1329
1330STATIC void
1331_xfs_buf_ioapply(
1332 struct xfs_buf *bp)
1333{
1334 struct blk_plug plug;
1335 int rw;
1336 int offset;
1337 int size;
1338 int i;
1339
1340
1341
1342
1343
1344 bp->b_error = 0;
1345
1346 if (bp->b_flags & XBF_WRITE) {
1347 if (bp->b_flags & XBF_SYNCIO)
1348 rw = WRITE_SYNC;
1349 else
1350 rw = WRITE;
1351 if (bp->b_flags & XBF_FUA)
1352 rw |= REQ_FUA;
1353 if (bp->b_flags & XBF_FLUSH)
1354 rw |= REQ_FLUSH;
1355
1356
1357
1358
1359
1360
1361 if (bp->b_ops) {
1362 bp->b_ops->verify_write(bp);
1363 if (bp->b_error) {
1364 xfs_force_shutdown(bp->b_target->bt_mount,
1365 SHUTDOWN_CORRUPT_INCORE);
1366 return;
1367 }
1368 }
1369 } else if (bp->b_flags & XBF_READ_AHEAD) {
1370 rw = READA;
1371 } else {
1372 rw = READ;
1373 }
1374
1375
1376 rw |= REQ_META;
1377
1378
1379
1380
1381
1382
1383
1384 offset = bp->b_offset;
1385 size = BBTOB(bp->b_io_length);
1386 blk_start_plug(&plug);
1387 for (i = 0; i < bp->b_map_count; i++) {
1388 xfs_buf_ioapply_map(bp, i, &offset, &size, rw);
1389 if (bp->b_error)
1390 break;
1391 if (size <= 0)
1392 break;
1393 }
1394 blk_finish_plug(&plug);
1395}
1396
1397void
1398xfs_buf_iorequest(
1399 xfs_buf_t *bp)
1400{
1401 trace_xfs_buf_iorequest(bp, _RET_IP_);
1402
1403 ASSERT(!(bp->b_flags & _XBF_DELWRI_Q));
1404
1405 if (bp->b_flags & XBF_WRITE)
1406 xfs_buf_wait_unpin(bp);
1407 xfs_buf_hold(bp);
1408
1409
1410
1411
1412
1413 atomic_set(&bp->b_io_remaining, 1);
1414 _xfs_buf_ioapply(bp);
1415 _xfs_buf_ioend(bp, 1);
1416
1417 xfs_buf_rele(bp);
1418}
1419
1420
1421
1422
1423
1424
1425int
1426xfs_buf_iowait(
1427 xfs_buf_t *bp)
1428{
1429 trace_xfs_buf_iowait(bp, _RET_IP_);
1430
1431 if (!bp->b_error)
1432 wait_for_completion(&bp->b_iowait);
1433
1434 trace_xfs_buf_iowait_done(bp, _RET_IP_);
1435 return bp->b_error;
1436}
1437
1438xfs_caddr_t
1439xfs_buf_offset(
1440 xfs_buf_t *bp,
1441 size_t offset)
1442{
1443 struct page *page;
1444
1445 if (bp->b_addr)
1446 return bp->b_addr + offset;
1447
1448 offset += bp->b_offset;
1449 page = bp->b_pages[offset >> PAGE_SHIFT];
1450 return (xfs_caddr_t)page_address(page) + (offset & (PAGE_SIZE-1));
1451}
1452
1453
1454
1455
1456void
1457xfs_buf_iomove(
1458 xfs_buf_t *bp,
1459 size_t boff,
1460 size_t bsize,
1461 void *data,
1462 xfs_buf_rw_t mode)
1463{
1464 size_t bend;
1465
1466 bend = boff + bsize;
1467 while (boff < bend) {
1468 struct page *page;
1469 int page_index, page_offset, csize;
1470
1471 page_index = (boff + bp->b_offset) >> PAGE_SHIFT;
1472 page_offset = (boff + bp->b_offset) & ~PAGE_MASK;
1473 page = bp->b_pages[page_index];
1474 csize = min_t(size_t, PAGE_SIZE - page_offset,
1475 BBTOB(bp->b_io_length) - boff);
1476
1477 ASSERT((csize + page_offset) <= PAGE_SIZE);
1478
1479 switch (mode) {
1480 case XBRW_ZERO:
1481 memset(page_address(page) + page_offset, 0, csize);
1482 break;
1483 case XBRW_READ:
1484 memcpy(data, page_address(page) + page_offset, csize);
1485 break;
1486 case XBRW_WRITE:
1487 memcpy(page_address(page) + page_offset, data, csize);
1488 }
1489
1490 boff += csize;
1491 data += csize;
1492 }
1493}
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504void
1505xfs_wait_buftarg(
1506 struct xfs_buftarg *btp)
1507{
1508 struct xfs_buf *bp;
1509
1510restart:
1511 spin_lock(&btp->bt_lru_lock);
1512 while (!list_empty(&btp->bt_lru)) {
1513 bp = list_first_entry(&btp->bt_lru, struct xfs_buf, b_lru);
1514 if (atomic_read(&bp->b_hold) > 1) {
1515 trace_xfs_buf_wait_buftarg(bp, _RET_IP_);
1516 list_move_tail(&bp->b_lru, &btp->bt_lru);
1517 spin_unlock(&btp->bt_lru_lock);
1518 delay(100);
1519 goto restart;
1520 }
1521
1522
1523
1524
1525 atomic_set(&bp->b_lru_ref, 0);
1526 spin_unlock(&btp->bt_lru_lock);
1527 xfs_buf_rele(bp);
1528 spin_lock(&btp->bt_lru_lock);
1529 }
1530 spin_unlock(&btp->bt_lru_lock);
1531}
1532
1533int
1534xfs_buftarg_shrink(
1535 struct shrinker *shrink,
1536 struct shrink_control *sc)
1537{
1538 struct xfs_buftarg *btp = container_of(shrink,
1539 struct xfs_buftarg, bt_shrinker);
1540 struct xfs_buf *bp;
1541 int nr_to_scan = sc->nr_to_scan;
1542 LIST_HEAD(dispose);
1543
1544 if (!nr_to_scan)
1545 return btp->bt_lru_nr;
1546
1547 spin_lock(&btp->bt_lru_lock);
1548 while (!list_empty(&btp->bt_lru)) {
1549 if (nr_to_scan-- <= 0)
1550 break;
1551
1552 bp = list_first_entry(&btp->bt_lru, struct xfs_buf, b_lru);
1553
1554
1555
1556
1557
1558
1559 if (!atomic_add_unless(&bp->b_lru_ref, -1, 0)) {
1560 list_move_tail(&bp->b_lru, &btp->bt_lru);
1561 continue;
1562 }
1563
1564
1565
1566
1567
1568 list_move(&bp->b_lru, &dispose);
1569 btp->bt_lru_nr--;
1570 bp->b_lru_flags |= _XBF_LRU_DISPOSE;
1571 }
1572 spin_unlock(&btp->bt_lru_lock);
1573
1574 while (!list_empty(&dispose)) {
1575 bp = list_first_entry(&dispose, struct xfs_buf, b_lru);
1576 list_del_init(&bp->b_lru);
1577 xfs_buf_rele(bp);
1578 }
1579
1580 return btp->bt_lru_nr;
1581}
1582
1583void
1584xfs_free_buftarg(
1585 struct xfs_mount *mp,
1586 struct xfs_buftarg *btp)
1587{
1588 unregister_shrinker(&btp->bt_shrinker);
1589
1590 if (mp->m_flags & XFS_MOUNT_BARRIER)
1591 xfs_blkdev_issue_flush(btp);
1592
1593 kmem_free(btp);
1594}
1595
1596STATIC int
1597xfs_setsize_buftarg_flags(
1598 xfs_buftarg_t *btp,
1599 unsigned int blocksize,
1600 unsigned int sectorsize,
1601 int verbose)
1602{
1603 btp->bt_bsize = blocksize;
1604 btp->bt_sshift = ffs(sectorsize) - 1;
1605 btp->bt_smask = sectorsize - 1;
1606
1607 if (set_blocksize(btp->bt_bdev, sectorsize)) {
1608 char name[BDEVNAME_SIZE];
1609
1610 bdevname(btp->bt_bdev, name);
1611
1612 xfs_warn(btp->bt_mount,
1613 "Cannot set_blocksize to %u on device %s\n",
1614 sectorsize, name);
1615 return EINVAL;
1616 }
1617
1618 return 0;
1619}
1620
1621
1622
1623
1624
1625
1626STATIC int
1627xfs_setsize_buftarg_early(
1628 xfs_buftarg_t *btp,
1629 struct block_device *bdev)
1630{
1631 return xfs_setsize_buftarg_flags(btp,
1632 PAGE_SIZE, bdev_logical_block_size(bdev), 0);
1633}
1634
1635int
1636xfs_setsize_buftarg(
1637 xfs_buftarg_t *btp,
1638 unsigned int blocksize,
1639 unsigned int sectorsize)
1640{
1641 return xfs_setsize_buftarg_flags(btp, blocksize, sectorsize, 1);
1642}
1643
1644xfs_buftarg_t *
1645xfs_alloc_buftarg(
1646 struct xfs_mount *mp,
1647 struct block_device *bdev,
1648 int external,
1649 const char *fsname)
1650{
1651 xfs_buftarg_t *btp;
1652
1653 btp = kmem_zalloc(sizeof(*btp), KM_SLEEP | KM_NOFS);
1654
1655 btp->bt_mount = mp;
1656 btp->bt_dev = bdev->bd_dev;
1657 btp->bt_bdev = bdev;
1658 btp->bt_bdi = blk_get_backing_dev_info(bdev);
1659 if (!btp->bt_bdi)
1660 goto error;
1661
1662 INIT_LIST_HEAD(&btp->bt_lru);
1663 spin_lock_init(&btp->bt_lru_lock);
1664 if (xfs_setsize_buftarg_early(btp, bdev))
1665 goto error;
1666 btp->bt_shrinker.shrink = xfs_buftarg_shrink;
1667 btp->bt_shrinker.seeks = DEFAULT_SEEKS;
1668 register_shrinker(&btp->bt_shrinker);
1669 return btp;
1670
1671error:
1672 kmem_free(btp);
1673 return NULL;
1674}
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687bool
1688xfs_buf_delwri_queue(
1689 struct xfs_buf *bp,
1690 struct list_head *list)
1691{
1692 ASSERT(xfs_buf_islocked(bp));
1693 ASSERT(!(bp->b_flags & XBF_READ));
1694
1695
1696
1697
1698
1699
1700 if (bp->b_flags & _XBF_DELWRI_Q) {
1701 trace_xfs_buf_delwri_queued(bp, _RET_IP_);
1702 return false;
1703 }
1704
1705 trace_xfs_buf_delwri_queue(bp, _RET_IP_);
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715 bp->b_flags |= _XBF_DELWRI_Q;
1716 if (list_empty(&bp->b_list)) {
1717 atomic_inc(&bp->b_hold);
1718 list_add_tail(&bp->b_list, list);
1719 }
1720
1721 return true;
1722}
1723
1724
1725
1726
1727
1728
1729static int
1730xfs_buf_cmp(
1731 void *priv,
1732 struct list_head *a,
1733 struct list_head *b)
1734{
1735 struct xfs_buf *ap = container_of(a, struct xfs_buf, b_list);
1736 struct xfs_buf *bp = container_of(b, struct xfs_buf, b_list);
1737 xfs_daddr_t diff;
1738
1739 diff = ap->b_maps[0].bm_bn - bp->b_maps[0].bm_bn;
1740 if (diff < 0)
1741 return -1;
1742 if (diff > 0)
1743 return 1;
1744 return 0;
1745}
1746
1747static int
1748__xfs_buf_delwri_submit(
1749 struct list_head *buffer_list,
1750 struct list_head *io_list,
1751 bool wait)
1752{
1753 struct blk_plug plug;
1754 struct xfs_buf *bp, *n;
1755 int pinned = 0;
1756
1757 list_for_each_entry_safe(bp, n, buffer_list, b_list) {
1758 if (!wait) {
1759 if (xfs_buf_ispinned(bp)) {
1760 pinned++;
1761 continue;
1762 }
1763 if (!xfs_buf_trylock(bp))
1764 continue;
1765 } else {
1766 xfs_buf_lock(bp);
1767 }
1768
1769
1770
1771
1772
1773
1774
1775 if (!(bp->b_flags & _XBF_DELWRI_Q)) {
1776 list_del_init(&bp->b_list);
1777 xfs_buf_relse(bp);
1778 continue;
1779 }
1780
1781 list_move_tail(&bp->b_list, io_list);
1782 trace_xfs_buf_delwri_split(bp, _RET_IP_);
1783 }
1784
1785 list_sort(NULL, io_list, xfs_buf_cmp);
1786
1787 blk_start_plug(&plug);
1788 list_for_each_entry_safe(bp, n, io_list, b_list) {
1789 bp->b_flags &= ~(_XBF_DELWRI_Q | XBF_ASYNC);
1790 bp->b_flags |= XBF_WRITE;
1791
1792 if (!wait) {
1793 bp->b_flags |= XBF_ASYNC;
1794 list_del_init(&bp->b_list);
1795 }
1796 xfs_bdstrat_cb(bp);
1797 }
1798 blk_finish_plug(&plug);
1799
1800 return pinned;
1801}
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812int
1813xfs_buf_delwri_submit_nowait(
1814 struct list_head *buffer_list)
1815{
1816 LIST_HEAD (io_list);
1817 return __xfs_buf_delwri_submit(buffer_list, &io_list, false);
1818}
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828int
1829xfs_buf_delwri_submit(
1830 struct list_head *buffer_list)
1831{
1832 LIST_HEAD (io_list);
1833 int error = 0, error2;
1834 struct xfs_buf *bp;
1835
1836 __xfs_buf_delwri_submit(buffer_list, &io_list, true);
1837
1838
1839 while (!list_empty(&io_list)) {
1840 bp = list_first_entry(&io_list, struct xfs_buf, b_list);
1841
1842 list_del_init(&bp->b_list);
1843 error2 = xfs_buf_iowait(bp);
1844 xfs_buf_relse(bp);
1845 if (!error)
1846 error = error2;
1847 }
1848
1849 return error;
1850}
1851
1852int __init
1853xfs_buf_init(void)
1854{
1855 xfs_buf_zone = kmem_zone_init_flags(sizeof(xfs_buf_t), "xfs_buf",
1856 KM_ZONE_HWALIGN, NULL);
1857 if (!xfs_buf_zone)
1858 goto out;
1859
1860 xfslogd_workqueue = alloc_workqueue("xfslogd",
1861 WQ_MEM_RECLAIM | WQ_HIGHPRI, 1);
1862 if (!xfslogd_workqueue)
1863 goto out_free_buf_zone;
1864
1865 return 0;
1866
1867 out_free_buf_zone:
1868 kmem_zone_destroy(xfs_buf_zone);
1869 out:
1870 return -ENOMEM;
1871}
1872
1873void
1874xfs_buf_terminate(void)
1875{
1876 destroy_workqueue(xfslogd_workqueue);
1877 kmem_zone_destroy(xfs_buf_zone);
1878}
1879