1
2
3
4
5#ifndef __LINUX_BIO_H
6#define __LINUX_BIO_H
7
8#include <linux/highmem.h>
9#include <linux/mempool.h>
10#include <linux/ioprio.h>
11
12#ifdef CONFIG_BLOCK
13
14#include <linux/blk_types.h>
15
16#define BIO_DEBUG
17
18#ifdef BIO_DEBUG
19#define BIO_BUG_ON BUG_ON
20#else
21#define BIO_BUG_ON
22#endif
23
24#define BIO_MAX_PAGES 256
25
26#define bio_prio(bio) (bio)->bi_ioprio
27#define bio_set_prio(bio, prio) ((bio)->bi_ioprio = prio)
28
29#define bio_iter_iovec(bio, iter) \
30 bvec_iter_bvec((bio)->bi_io_vec, (iter))
31
32#define bio_iter_page(bio, iter) \
33 bvec_iter_page((bio)->bi_io_vec, (iter))
34#define bio_iter_len(bio, iter) \
35 bvec_iter_len((bio)->bi_io_vec, (iter))
36#define bio_iter_offset(bio, iter) \
37 bvec_iter_offset((bio)->bi_io_vec, (iter))
38
39#define bio_page(bio) bio_iter_page((bio), (bio)->bi_iter)
40#define bio_offset(bio) bio_iter_offset((bio), (bio)->bi_iter)
41#define bio_iovec(bio) bio_iter_iovec((bio), (bio)->bi_iter)
42
43#define bio_multiple_segments(bio) \
44 ((bio)->bi_iter.bi_size != bio_iovec(bio).bv_len)
45
46#define bvec_iter_sectors(iter) ((iter).bi_size >> 9)
47#define bvec_iter_end_sector(iter) ((iter).bi_sector + bvec_iter_sectors((iter)))
48
49#define bio_sectors(bio) bvec_iter_sectors((bio)->bi_iter)
50#define bio_end_sector(bio) bvec_iter_end_sector((bio)->bi_iter)
51
52
53
54
55#define bio_data_dir(bio) \
56 (op_is_write(bio_op(bio)) ? WRITE : READ)
57
58
59
60
61static inline bool bio_has_data(struct bio *bio)
62{
63 if (bio &&
64 bio->bi_iter.bi_size &&
65 bio_op(bio) != REQ_OP_DISCARD &&
66 bio_op(bio) != REQ_OP_SECURE_ERASE &&
67 bio_op(bio) != REQ_OP_WRITE_ZEROES)
68 return true;
69
70 return false;
71}
72
73static inline bool bio_no_advance_iter(struct bio *bio)
74{
75 return bio_op(bio) == REQ_OP_DISCARD ||
76 bio_op(bio) == REQ_OP_SECURE_ERASE ||
77 bio_op(bio) == REQ_OP_WRITE_SAME ||
78 bio_op(bio) == REQ_OP_WRITE_ZEROES;
79}
80
81static inline bool bio_mergeable(struct bio *bio)
82{
83 if (bio->bi_opf & REQ_NOMERGE_FLAGS)
84 return false;
85
86 return true;
87}
88
89static inline unsigned int bio_cur_bytes(struct bio *bio)
90{
91 if (bio_has_data(bio))
92 return bio_iovec(bio).bv_len;
93 else
94 return bio->bi_iter.bi_size;
95}
96
97static inline void *bio_data(struct bio *bio)
98{
99 if (bio_has_data(bio))
100 return page_address(bio_page(bio)) + bio_offset(bio);
101
102 return NULL;
103}
104
105static inline bool bio_full(struct bio *bio)
106{
107 return bio->bi_vcnt >= bio->bi_max_vecs;
108}
109
110static inline bool bio_next_segment(const struct bio *bio,
111 struct bvec_iter_all *iter)
112{
113 if (iter->idx >= bio->bi_vcnt)
114 return false;
115
116 bvec_advance(&bio->bi_io_vec[iter->idx], iter);
117 return true;
118}
119
120
121
122
123
124#define bio_for_each_segment_all(bvl, bio, iter) \
125 for (bvl = bvec_init_iter_all(&iter); bio_next_segment((bio), &iter); )
126
127static inline void bio_advance_iter(struct bio *bio, struct bvec_iter *iter,
128 unsigned bytes)
129{
130 iter->bi_sector += bytes >> 9;
131
132 if (bio_no_advance_iter(bio))
133 iter->bi_size -= bytes;
134 else
135 bvec_iter_advance(bio->bi_io_vec, iter, bytes);
136
137}
138
139#define __bio_for_each_segment(bvl, bio, iter, start) \
140 for (iter = (start); \
141 (iter).bi_size && \
142 ((bvl = bio_iter_iovec((bio), (iter))), 1); \
143 bio_advance_iter((bio), &(iter), (bvl).bv_len))
144
145#define bio_for_each_segment(bvl, bio, iter) \
146 __bio_for_each_segment(bvl, bio, iter, (bio)->bi_iter)
147
148#define __bio_for_each_bvec(bvl, bio, iter, start) \
149 for (iter = (start); \
150 (iter).bi_size && \
151 ((bvl = mp_bvec_iter_bvec((bio)->bi_io_vec, (iter))), 1); \
152 bio_advance_iter((bio), &(iter), (bvl).bv_len))
153
154
155#define bio_for_each_bvec(bvl, bio, iter) \
156 __bio_for_each_bvec(bvl, bio, iter, (bio)->bi_iter)
157
158#define bio_iter_last(bvec, iter) ((iter).bi_size == (bvec).bv_len)
159
160static inline unsigned bio_segments(struct bio *bio)
161{
162 unsigned segs = 0;
163 struct bio_vec bv;
164 struct bvec_iter iter;
165
166
167
168
169
170
171 switch (bio_op(bio)) {
172 case REQ_OP_DISCARD:
173 case REQ_OP_SECURE_ERASE:
174 case REQ_OP_WRITE_ZEROES:
175 return 0;
176 case REQ_OP_WRITE_SAME:
177 return 1;
178 default:
179 break;
180 }
181
182 bio_for_each_segment(bv, bio, iter)
183 segs++;
184
185 return segs;
186}
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202static inline void bio_get(struct bio *bio)
203{
204 bio->bi_flags |= (1 << BIO_REFFED);
205 smp_mb__before_atomic();
206 atomic_inc(&bio->__bi_cnt);
207}
208
209static inline void bio_cnt_set(struct bio *bio, unsigned int count)
210{
211 if (count != 1) {
212 bio->bi_flags |= (1 << BIO_REFFED);
213 smp_mb();
214 }
215 atomic_set(&bio->__bi_cnt, count);
216}
217
218static inline bool bio_flagged(struct bio *bio, unsigned int bit)
219{
220 return (bio->bi_flags & (1U << bit)) != 0;
221}
222
223static inline void bio_set_flag(struct bio *bio, unsigned int bit)
224{
225 bio->bi_flags |= (1U << bit);
226}
227
228static inline void bio_clear_flag(struct bio *bio, unsigned int bit)
229{
230 bio->bi_flags &= ~(1U << bit);
231}
232
233static inline void bio_get_first_bvec(struct bio *bio, struct bio_vec *bv)
234{
235 *bv = bio_iovec(bio);
236}
237
238static inline void bio_get_last_bvec(struct bio *bio, struct bio_vec *bv)
239{
240 struct bvec_iter iter = bio->bi_iter;
241 int idx;
242
243 if (unlikely(!bio_multiple_segments(bio))) {
244 *bv = bio_iovec(bio);
245 return;
246 }
247
248 bio_advance_iter(bio, &iter, iter.bi_size);
249
250 if (!iter.bi_bvec_done)
251 idx = iter.bi_idx - 1;
252 else
253 idx = iter.bi_idx;
254
255 *bv = bio->bi_io_vec[idx];
256
257
258
259
260
261 if (iter.bi_bvec_done)
262 bv->bv_len = iter.bi_bvec_done;
263}
264
265static inline struct bio_vec *bio_first_bvec_all(struct bio *bio)
266{
267 WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));
268 return bio->bi_io_vec;
269}
270
271static inline struct page *bio_first_page_all(struct bio *bio)
272{
273 return bio_first_bvec_all(bio)->bv_page;
274}
275
276static inline struct bio_vec *bio_last_bvec_all(struct bio *bio)
277{
278 WARN_ON_ONCE(bio_flagged(bio, BIO_CLONED));
279 return &bio->bi_io_vec[bio->bi_vcnt - 1];
280}
281
282enum bip_flags {
283 BIP_BLOCK_INTEGRITY = 1 << 0,
284 BIP_MAPPED_INTEGRITY = 1 << 1,
285 BIP_CTRL_NOCHECK = 1 << 2,
286 BIP_DISK_NOCHECK = 1 << 3,
287 BIP_IP_CHECKSUM = 1 << 4,
288};
289
290
291
292
293struct bio_integrity_payload {
294 struct bio *bip_bio;
295
296 struct bvec_iter bip_iter;
297
298 unsigned short bip_slab;
299 unsigned short bip_vcnt;
300 unsigned short bip_max_vcnt;
301 unsigned short bip_flags;
302
303 struct bvec_iter bio_iter;
304
305 struct work_struct bip_work;
306
307 struct bio_vec *bip_vec;
308 struct bio_vec bip_inline_vecs[0];
309};
310
311#if defined(CONFIG_BLK_DEV_INTEGRITY)
312
313static inline struct bio_integrity_payload *bio_integrity(struct bio *bio)
314{
315 if (bio->bi_opf & REQ_INTEGRITY)
316 return bio->bi_integrity;
317
318 return NULL;
319}
320
321static inline bool bio_integrity_flagged(struct bio *bio, enum bip_flags flag)
322{
323 struct bio_integrity_payload *bip = bio_integrity(bio);
324
325 if (bip)
326 return bip->bip_flags & flag;
327
328 return false;
329}
330
331static inline sector_t bip_get_seed(struct bio_integrity_payload *bip)
332{
333 return bip->bip_iter.bi_sector;
334}
335
336static inline void bip_set_seed(struct bio_integrity_payload *bip,
337 sector_t seed)
338{
339 bip->bip_iter.bi_sector = seed;
340}
341
342#endif
343
344extern void bio_trim(struct bio *bio, int offset, int size);
345extern struct bio *bio_split(struct bio *bio, int sectors,
346 gfp_t gfp, struct bio_set *bs);
347
348
349
350
351
352
353
354
355
356
357
358static inline struct bio *bio_next_split(struct bio *bio, int sectors,
359 gfp_t gfp, struct bio_set *bs)
360{
361 if (sectors >= bio_sectors(bio))
362 return bio;
363
364 return bio_split(bio, sectors, gfp, bs);
365}
366
367enum {
368 BIOSET_NEED_BVECS = BIT(0),
369 BIOSET_NEED_RESCUER = BIT(1),
370};
371extern int bioset_init(struct bio_set *, unsigned int, unsigned int, int flags);
372extern void bioset_exit(struct bio_set *);
373extern int biovec_init_pool(mempool_t *pool, int pool_entries);
374extern int bioset_init_from_src(struct bio_set *bs, struct bio_set *src);
375
376extern struct bio *bio_alloc_bioset(gfp_t, unsigned int, struct bio_set *);
377extern void bio_put(struct bio *);
378
379extern void __bio_clone_fast(struct bio *, struct bio *);
380extern struct bio *bio_clone_fast(struct bio *, gfp_t, struct bio_set *);
381
382extern struct bio_set fs_bio_set;
383
384static inline struct bio *bio_alloc(gfp_t gfp_mask, unsigned int nr_iovecs)
385{
386 return bio_alloc_bioset(gfp_mask, nr_iovecs, &fs_bio_set);
387}
388
389static inline struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned int nr_iovecs)
390{
391 return bio_alloc_bioset(gfp_mask, nr_iovecs, NULL);
392}
393
394extern blk_qc_t submit_bio(struct bio *);
395
396extern void bio_endio(struct bio *);
397
398static inline void bio_io_error(struct bio *bio)
399{
400 bio->bi_status = BLK_STS_IOERR;
401 bio_endio(bio);
402}
403
404static inline void bio_wouldblock_error(struct bio *bio)
405{
406 bio->bi_status = BLK_STS_AGAIN;
407 bio_endio(bio);
408}
409
410struct request_queue;
411extern int bio_phys_segments(struct request_queue *, struct bio *);
412
413extern int submit_bio_wait(struct bio *bio);
414extern void bio_advance(struct bio *, unsigned);
415
416extern void bio_init(struct bio *bio, struct bio_vec *table,
417 unsigned short max_vecs);
418extern void bio_uninit(struct bio *);
419extern void bio_reset(struct bio *);
420void bio_chain(struct bio *, struct bio *);
421
422extern int bio_add_page(struct bio *, struct page *, unsigned int,unsigned int);
423extern int bio_add_pc_page(struct request_queue *, struct bio *, struct page *,
424 unsigned int, unsigned int);
425bool __bio_try_merge_page(struct bio *bio, struct page *page,
426 unsigned int len, unsigned int off, bool *same_page);
427void __bio_add_page(struct bio *bio, struct page *page,
428 unsigned int len, unsigned int off);
429int bio_iov_iter_get_pages(struct bio *bio, struct iov_iter *iter);
430struct rq_map_data;
431extern struct bio *bio_map_user_iov(struct request_queue *,
432 struct iov_iter *, gfp_t);
433extern void bio_unmap_user(struct bio *);
434extern struct bio *bio_map_kern(struct request_queue *, void *, unsigned int,
435 gfp_t);
436extern struct bio *bio_copy_kern(struct request_queue *, void *, unsigned int,
437 gfp_t, int);
438extern void bio_set_pages_dirty(struct bio *bio);
439extern void bio_check_pages_dirty(struct bio *bio);
440
441void generic_start_io_acct(struct request_queue *q, int op,
442 unsigned long sectors, struct hd_struct *part);
443void generic_end_io_acct(struct request_queue *q, int op,
444 struct hd_struct *part,
445 unsigned long start_time);
446
447#ifndef ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
448# error "You should define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE for your platform"
449#endif
450#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
451extern void bio_flush_dcache_pages(struct bio *bi);
452#else
453static inline void bio_flush_dcache_pages(struct bio *bi)
454{
455}
456#endif
457
458extern void bio_copy_data_iter(struct bio *dst, struct bvec_iter *dst_iter,
459 struct bio *src, struct bvec_iter *src_iter);
460extern void bio_copy_data(struct bio *dst, struct bio *src);
461extern void bio_list_copy_data(struct bio *dst, struct bio *src);
462extern void bio_free_pages(struct bio *bio);
463
464extern struct bio *bio_copy_user_iov(struct request_queue *,
465 struct rq_map_data *,
466 struct iov_iter *,
467 gfp_t);
468extern int bio_uncopy_user(struct bio *);
469void zero_fill_bio_iter(struct bio *bio, struct bvec_iter iter);
470
471static inline void zero_fill_bio(struct bio *bio)
472{
473 zero_fill_bio_iter(bio, bio->bi_iter);
474}
475
476extern struct bio_vec *bvec_alloc(gfp_t, int, unsigned long *, mempool_t *);
477extern void bvec_free(mempool_t *, struct bio_vec *, unsigned int);
478extern unsigned int bvec_nr_vecs(unsigned short idx);
479extern const char *bio_devname(struct bio *bio, char *buffer);
480
481#define bio_set_dev(bio, bdev) \
482do { \
483 if ((bio)->bi_disk != (bdev)->bd_disk) \
484 bio_clear_flag(bio, BIO_THROTTLED);\
485 (bio)->bi_disk = (bdev)->bd_disk; \
486 (bio)->bi_partno = (bdev)->bd_partno; \
487 bio_associate_blkg(bio); \
488} while (0)
489
490#define bio_copy_dev(dst, src) \
491do { \
492 (dst)->bi_disk = (src)->bi_disk; \
493 (dst)->bi_partno = (src)->bi_partno; \
494 bio_clone_blkg_association(dst, src); \
495} while (0)
496
497#define bio_dev(bio) \
498 disk_devt((bio)->bi_disk)
499
500#if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP)
501void bio_associate_blkg_from_page(struct bio *bio, struct page *page);
502#else
503static inline void bio_associate_blkg_from_page(struct bio *bio,
504 struct page *page) { }
505#endif
506
507#ifdef CONFIG_BLK_CGROUP
508void bio_disassociate_blkg(struct bio *bio);
509void bio_associate_blkg(struct bio *bio);
510void bio_associate_blkg_from_css(struct bio *bio,
511 struct cgroup_subsys_state *css);
512void bio_clone_blkg_association(struct bio *dst, struct bio *src);
513#else
514static inline void bio_disassociate_blkg(struct bio *bio) { }
515static inline void bio_associate_blkg(struct bio *bio) { }
516static inline void bio_associate_blkg_from_css(struct bio *bio,
517 struct cgroup_subsys_state *css)
518{ }
519static inline void bio_clone_blkg_association(struct bio *dst,
520 struct bio *src) { }
521#endif
522
523#ifdef CONFIG_HIGHMEM
524
525
526
527
528static inline char *bvec_kmap_irq(struct bio_vec *bvec, unsigned long *flags)
529{
530 unsigned long addr;
531
532
533
534
535
536 local_irq_save(*flags);
537 addr = (unsigned long) kmap_atomic(bvec->bv_page);
538
539 BUG_ON(addr & ~PAGE_MASK);
540
541 return (char *) addr + bvec->bv_offset;
542}
543
544static inline void bvec_kunmap_irq(char *buffer, unsigned long *flags)
545{
546 unsigned long ptr = (unsigned long) buffer & PAGE_MASK;
547
548 kunmap_atomic((void *) ptr);
549 local_irq_restore(*flags);
550}
551
552#else
553static inline char *bvec_kmap_irq(struct bio_vec *bvec, unsigned long *flags)
554{
555 return page_address(bvec->bv_page) + bvec->bv_offset;
556}
557
558static inline void bvec_kunmap_irq(char *buffer, unsigned long *flags)
559{
560 *flags = 0;
561}
562#endif
563
564
565
566
567
568
569
570
571struct bio_list {
572 struct bio *head;
573 struct bio *tail;
574};
575
576static inline int bio_list_empty(const struct bio_list *bl)
577{
578 return bl->head == NULL;
579}
580
581static inline void bio_list_init(struct bio_list *bl)
582{
583 bl->head = bl->tail = NULL;
584}
585
586#define BIO_EMPTY_LIST { NULL, NULL }
587
588#define bio_list_for_each(bio, bl) \
589 for (bio = (bl)->head; bio; bio = bio->bi_next)
590
591static inline unsigned bio_list_size(const struct bio_list *bl)
592{
593 unsigned sz = 0;
594 struct bio *bio;
595
596 bio_list_for_each(bio, bl)
597 sz++;
598
599 return sz;
600}
601
602static inline void bio_list_add(struct bio_list *bl, struct bio *bio)
603{
604 bio->bi_next = NULL;
605
606 if (bl->tail)
607 bl->tail->bi_next = bio;
608 else
609 bl->head = bio;
610
611 bl->tail = bio;
612}
613
614static inline void bio_list_add_head(struct bio_list *bl, struct bio *bio)
615{
616 bio->bi_next = bl->head;
617
618 bl->head = bio;
619
620 if (!bl->tail)
621 bl->tail = bio;
622}
623
624static inline void bio_list_merge(struct bio_list *bl, struct bio_list *bl2)
625{
626 if (!bl2->head)
627 return;
628
629 if (bl->tail)
630 bl->tail->bi_next = bl2->head;
631 else
632 bl->head = bl2->head;
633
634 bl->tail = bl2->tail;
635}
636
637static inline void bio_list_merge_head(struct bio_list *bl,
638 struct bio_list *bl2)
639{
640 if (!bl2->head)
641 return;
642
643 if (bl->head)
644 bl2->tail->bi_next = bl->head;
645 else
646 bl->tail = bl2->tail;
647
648 bl->head = bl2->head;
649}
650
651static inline struct bio *bio_list_peek(struct bio_list *bl)
652{
653 return bl->head;
654}
655
656static inline struct bio *bio_list_pop(struct bio_list *bl)
657{
658 struct bio *bio = bl->head;
659
660 if (bio) {
661 bl->head = bl->head->bi_next;
662 if (!bl->head)
663 bl->tail = NULL;
664
665 bio->bi_next = NULL;
666 }
667
668 return bio;
669}
670
671static inline struct bio *bio_list_get(struct bio_list *bl)
672{
673 struct bio *bio = bl->head;
674
675 bl->head = bl->tail = NULL;
676
677 return bio;
678}
679
680
681
682
683
684static inline void bio_inc_remaining(struct bio *bio)
685{
686 bio_set_flag(bio, BIO_CHAIN);
687 smp_mb__before_atomic();
688 atomic_inc(&bio->__bi_remaining);
689}
690
691
692
693
694
695
696
697#define BIO_POOL_SIZE 2
698
699struct bio_set {
700 struct kmem_cache *bio_slab;
701 unsigned int front_pad;
702
703 mempool_t bio_pool;
704 mempool_t bvec_pool;
705#if defined(CONFIG_BLK_DEV_INTEGRITY)
706 mempool_t bio_integrity_pool;
707 mempool_t bvec_integrity_pool;
708#endif
709
710
711
712
713
714 spinlock_t rescue_lock;
715 struct bio_list rescue_list;
716 struct work_struct rescue_work;
717 struct workqueue_struct *rescue_workqueue;
718};
719
720struct biovec_slab {
721 int nr_vecs;
722 char *name;
723 struct kmem_cache *slab;
724};
725
726static inline bool bioset_initialized(struct bio_set *bs)
727{
728 return bs->bio_slab != NULL;
729}
730
731
732
733
734
735#define BIO_SPLIT_ENTRIES 2
736
737#if defined(CONFIG_BLK_DEV_INTEGRITY)
738
739#define bip_for_each_vec(bvl, bip, iter) \
740 for_each_bvec(bvl, (bip)->bip_vec, iter, (bip)->bip_iter)
741
742#define bio_for_each_integrity_vec(_bvl, _bio, _iter) \
743 for_each_bio(_bio) \
744 bip_for_each_vec(_bvl, _bio->bi_integrity, _iter)
745
746extern struct bio_integrity_payload *bio_integrity_alloc(struct bio *, gfp_t, unsigned int);
747extern int bio_integrity_add_page(struct bio *, struct page *, unsigned int, unsigned int);
748extern bool bio_integrity_prep(struct bio *);
749extern void bio_integrity_advance(struct bio *, unsigned int);
750extern void bio_integrity_trim(struct bio *);
751extern int bio_integrity_clone(struct bio *, struct bio *, gfp_t);
752extern int bioset_integrity_create(struct bio_set *, int);
753extern void bioset_integrity_free(struct bio_set *);
754extern void bio_integrity_init(void);
755
756#else
757
758static inline void *bio_integrity(struct bio *bio)
759{
760 return NULL;
761}
762
763static inline int bioset_integrity_create(struct bio_set *bs, int pool_size)
764{
765 return 0;
766}
767
768static inline void bioset_integrity_free (struct bio_set *bs)
769{
770 return;
771}
772
773static inline bool bio_integrity_prep(struct bio *bio)
774{
775 return true;
776}
777
778static inline int bio_integrity_clone(struct bio *bio, struct bio *bio_src,
779 gfp_t gfp_mask)
780{
781 return 0;
782}
783
784static inline void bio_integrity_advance(struct bio *bio,
785 unsigned int bytes_done)
786{
787 return;
788}
789
790static inline void bio_integrity_trim(struct bio *bio)
791{
792 return;
793}
794
795static inline void bio_integrity_init(void)
796{
797 return;
798}
799
800static inline bool bio_integrity_flagged(struct bio *bio, enum bip_flags flag)
801{
802 return false;
803}
804
805static inline void *bio_integrity_alloc(struct bio * bio, gfp_t gfp,
806 unsigned int nr)
807{
808 return ERR_PTR(-EINVAL);
809}
810
811static inline int bio_integrity_add_page(struct bio *bio, struct page *page,
812 unsigned int len, unsigned int offset)
813{
814 return 0;
815}
816
817#endif
818
819
820
821
822
823
824
825
826static inline void bio_set_polled(struct bio *bio, struct kiocb *kiocb)
827{
828 bio->bi_opf |= REQ_HIPRI;
829 if (!is_sync_kiocb(kiocb))
830 bio->bi_opf |= REQ_NOWAIT;
831}
832
833#endif
834#endif
835