1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20#ifndef __LINUX_BIO_H
21#define __LINUX_BIO_H
22
23#include <linux/highmem.h>
24#include <linux/mempool.h>
25#include <linux/ioprio.h>
26#include <linux/bug.h>
27
28#ifdef CONFIG_BLOCK
29
30#include <asm/io.h>
31
32
33#include <linux/blk_types.h>
34
35#define BIO_DEBUG
36
37#ifdef BIO_DEBUG
38#define BIO_BUG_ON BUG_ON
39#else
40#define BIO_BUG_ON
41#endif
42
43#define BIO_MAX_PAGES 256
44#define BIO_MAX_SIZE (BIO_MAX_PAGES << PAGE_CACHE_SHIFT)
45#define BIO_MAX_SECTORS (BIO_MAX_SIZE >> 9)
46
47
48
49
50#define BIO_PRIO_SHIFT (8 * sizeof(unsigned long) - IOPRIO_BITS)
51#define bio_prio(bio) ((bio)->bi_rw >> BIO_PRIO_SHIFT)
52#define bio_prio_valid(bio) ioprio_valid(bio_prio(bio))
53
54#define bio_set_prio(bio, prio) do { \
55 WARN_ON(prio >= (1 << IOPRIO_BITS)); \
56 (bio)->bi_rw &= ((1UL << BIO_PRIO_SHIFT) - 1); \
57 (bio)->bi_rw |= ((unsigned long) (prio) << BIO_PRIO_SHIFT); \
58} while (0)
59
60
61
62
63
64#define bio_iovec_idx(bio, idx) (&((bio)->bi_io_vec[(idx)]))
65#define bio_iovec(bio) bio_iovec_idx((bio), (bio)->bi_idx)
66#define bio_page(bio) bio_iovec((bio))->bv_page
67#define bio_offset(bio) bio_iovec((bio))->bv_offset
68#define bio_segments(bio) ((bio)->bi_vcnt - (bio)->bi_idx)
69#define bio_sectors(bio) ((bio)->bi_size >> 9)
70#define bio_end_sector(bio) ((bio)->bi_sector + bio_sectors((bio)))
71
72static inline unsigned int bio_cur_bytes(struct bio *bio)
73{
74 if (bio->bi_vcnt)
75 return bio_iovec(bio)->bv_len;
76 else
77 return bio->bi_size;
78}
79
80static inline void *bio_data(struct bio *bio)
81{
82 if (bio->bi_vcnt)
83 return page_address(bio_page(bio)) + bio_offset(bio);
84
85 return NULL;
86}
87
88
89
90
91#define bio_to_phys(bio) (page_to_phys(bio_page((bio))) + (unsigned long) bio_offset((bio)))
92#define bvec_to_phys(bv) (page_to_phys((bv)->bv_page) + (unsigned long) (bv)->bv_offset)
93
94
95
96
97
98
99
100#define __bio_kmap_atomic(bio, idx, kmtype) \
101 (kmap_atomic(bio_iovec_idx((bio), (idx))->bv_page) + \
102 bio_iovec_idx((bio), (idx))->bv_offset)
103
104#define __bio_kunmap_atomic(addr, kmtype) kunmap_atomic(addr)
105
106
107
108
109
110#define __BVEC_END(bio) bio_iovec_idx((bio), (bio)->bi_vcnt - 1)
111#define __BVEC_START(bio) bio_iovec_idx((bio), (bio)->bi_idx)
112
113
114#define __BIOVEC_PHYS_MERGEABLE(vec1, vec2) \
115 ((bvec_to_phys((vec1)) + (vec1)->bv_len) == bvec_to_phys((vec2)))
116
117
118
119
120#ifndef BIOVEC_PHYS_MERGEABLE
121#define BIOVEC_PHYS_MERGEABLE(vec1, vec2) \
122 __BIOVEC_PHYS_MERGEABLE(vec1, vec2)
123#endif
124
125#define __BIO_SEG_BOUNDARY(addr1, addr2, mask) \
126 (((addr1) | (mask)) == (((addr2) - 1) | (mask)))
127#define BIOVEC_SEG_BOUNDARY(q, b1, b2) \
128 __BIO_SEG_BOUNDARY(bvec_to_phys((b1)), bvec_to_phys((b2)) + (b2)->bv_len, queue_segment_boundary((q)))
129#define BIO_SEG_BOUNDARY(q, b1, b2) \
130 BIOVEC_SEG_BOUNDARY((q), __BVEC_END((b1)), __BVEC_START((b2)))
131
132#define bio_io_error(bio) bio_endio((bio), -EIO)
133
134
135
136
137
138#define __bio_for_each_segment(bvl, bio, i, start_idx) \
139 for (bvl = bio_iovec_idx((bio), (start_idx)), i = (start_idx); \
140 i < (bio)->bi_vcnt; \
141 bvl++, i++)
142
143
144
145
146
147#define bio_for_each_segment_all(bvl, bio, i) \
148 for (i = 0; \
149 bvl = bio_iovec_idx((bio), (i)), i < (bio)->bi_vcnt; \
150 i++)
151
152#define bio_for_each_segment(bvl, bio, i) \
153 for (i = (bio)->bi_idx; \
154 bvl = bio_iovec_idx((bio), (i)), i < (bio)->bi_vcnt; \
155 i++)
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171#define bio_get(bio) atomic_inc(&(bio)->bi_cnt)
172
173#if defined(CONFIG_BLK_DEV_INTEGRITY)
174
175
176
177struct bio_integrity_payload {
178 struct bio *bip_bio;
179
180 sector_t bip_sector;
181
182 void *bip_buf;
183 bio_end_io_t *bip_end_io;
184
185 unsigned int bip_size;
186
187 unsigned short bip_slab;
188 unsigned short bip_vcnt;
189 unsigned short bip_idx;
190 unsigned bip_owns_buf:1;
191
192 struct work_struct bip_work;
193
194 struct bio_vec *bip_vec;
195 struct bio_vec bip_inline_vecs[0];
196};
197#endif
198
199
200
201
202
203
204
205
206
207
208
209struct bio_pair {
210 struct bio bio1, bio2;
211 struct bio_vec bv1, bv2;
212#if defined(CONFIG_BLK_DEV_INTEGRITY)
213 struct bio_integrity_payload bip1, bip2;
214 struct bio_vec iv1, iv2;
215#endif
216 atomic_t cnt;
217 int error;
218};
219extern struct bio_pair *bio_split(struct bio *bi, int first_sectors);
220extern void bio_pair_release(struct bio_pair *dbio);
221
222extern struct bio_set *bioset_create(unsigned int, unsigned int);
223extern void bioset_free(struct bio_set *);
224extern mempool_t *biovec_create_pool(struct bio_set *bs, int pool_entries);
225
226extern struct bio *bio_alloc_bioset(gfp_t, int, struct bio_set *);
227extern void bio_put(struct bio *);
228
229extern void __bio_clone(struct bio *, struct bio *);
230extern struct bio *bio_clone_bioset(struct bio *, gfp_t, struct bio_set *bs);
231
232extern struct bio_set *fs_bio_set;
233
234static inline struct bio *bio_alloc(gfp_t gfp_mask, unsigned int nr_iovecs)
235{
236 return bio_alloc_bioset(gfp_mask, nr_iovecs, fs_bio_set);
237}
238
239static inline struct bio *bio_clone(struct bio *bio, gfp_t gfp_mask)
240{
241 return bio_clone_bioset(bio, gfp_mask, fs_bio_set);
242}
243
244static inline struct bio *bio_kmalloc(gfp_t gfp_mask, unsigned int nr_iovecs)
245{
246 return bio_alloc_bioset(gfp_mask, nr_iovecs, NULL);
247}
248
249static inline struct bio *bio_clone_kmalloc(struct bio *bio, gfp_t gfp_mask)
250{
251 return bio_clone_bioset(bio, gfp_mask, NULL);
252
253}
254
255extern void bio_endio(struct bio *, int);
256struct request_queue;
257extern int bio_phys_segments(struct request_queue *, struct bio *);
258
259extern int submit_bio_wait(int rw, struct bio *bio);
260extern void bio_advance(struct bio *, unsigned);
261
262extern void bio_init(struct bio *);
263extern void bio_reset(struct bio *);
264
265extern int bio_add_page(struct bio *, struct page *, unsigned int,unsigned int);
266extern int bio_add_pc_page(struct request_queue *, struct bio *, struct page *,
267 unsigned int, unsigned int);
268extern int bio_get_nr_vecs(struct block_device *);
269extern sector_t bio_sector_offset(struct bio *, unsigned short, unsigned int);
270extern struct bio *bio_map_user(struct request_queue *, struct block_device *,
271 unsigned long, unsigned int, int, gfp_t);
272struct sg_iovec;
273struct rq_map_data;
274extern struct bio *bio_map_user_iov(struct request_queue *,
275 struct block_device *,
276 struct sg_iovec *, int, int, gfp_t);
277extern void bio_unmap_user(struct bio *);
278extern struct bio *bio_map_kern(struct request_queue *, void *, unsigned int,
279 gfp_t);
280extern struct bio *bio_copy_kern(struct request_queue *, void *, unsigned int,
281 gfp_t, int);
282extern void bio_set_pages_dirty(struct bio *bio);
283extern void bio_check_pages_dirty(struct bio *bio);
284
285#ifndef ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
286# error "You should define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE for your platform"
287#endif
288#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
289extern void bio_flush_dcache_pages(struct bio *bi);
290#else
291static inline void bio_flush_dcache_pages(struct bio *bi)
292{
293}
294#endif
295
296extern void bio_copy_data(struct bio *dst, struct bio *src);
297extern int bio_alloc_pages(struct bio *bio, gfp_t gfp);
298
299extern struct bio *bio_copy_user(struct request_queue *, struct rq_map_data *,
300 unsigned long, unsigned int, int, gfp_t);
301extern struct bio *bio_copy_user_iov(struct request_queue *,
302 struct rq_map_data *, struct sg_iovec *,
303 int, int, gfp_t);
304extern int bio_uncopy_user(struct bio *);
305void zero_fill_bio(struct bio *bio);
306extern struct bio_vec *bvec_alloc(gfp_t, int, unsigned long *, mempool_t *);
307extern void bvec_free(mempool_t *, struct bio_vec *, unsigned int);
308extern unsigned int bvec_nr_vecs(unsigned short idx);
309
310#ifdef CONFIG_BLK_CGROUP
311int bio_associate_current(struct bio *bio);
312void bio_disassociate_task(struct bio *bio);
313#else
314static inline int bio_associate_current(struct bio *bio) { return -ENOENT; }
315static inline void bio_disassociate_task(struct bio *bio) { }
316#endif
317
318#ifdef CONFIG_HIGHMEM
319
320
321
322
323static inline char *bvec_kmap_irq(struct bio_vec *bvec, unsigned long *flags)
324{
325 unsigned long addr;
326
327
328
329
330
331 local_irq_save(*flags);
332 addr = (unsigned long) kmap_atomic(bvec->bv_page);
333
334 BUG_ON(addr & ~PAGE_MASK);
335
336 return (char *) addr + bvec->bv_offset;
337}
338
339static inline void bvec_kunmap_irq(char *buffer, unsigned long *flags)
340{
341 unsigned long ptr = (unsigned long) buffer & PAGE_MASK;
342
343 kunmap_atomic((void *) ptr);
344 local_irq_restore(*flags);
345}
346
347#else
348static inline char *bvec_kmap_irq(struct bio_vec *bvec, unsigned long *flags)
349{
350 return page_address(bvec->bv_page) + bvec->bv_offset;
351}
352
353static inline void bvec_kunmap_irq(char *buffer, unsigned long *flags)
354{
355 *flags = 0;
356}
357#endif
358
359static inline char *__bio_kmap_irq(struct bio *bio, unsigned short idx,
360 unsigned long *flags)
361{
362 return bvec_kmap_irq(bio_iovec_idx(bio, idx), flags);
363}
364#define __bio_kunmap_irq(buf, flags) bvec_kunmap_irq(buf, flags)
365
366#define bio_kmap_irq(bio, flags) \
367 __bio_kmap_irq((bio), (bio)->bi_idx, (flags))
368#define bio_kunmap_irq(buf,flags) __bio_kunmap_irq(buf, flags)
369
370
371
372
373static inline bool bio_has_data(struct bio *bio)
374{
375 if (bio && bio->bi_vcnt)
376 return true;
377
378 return false;
379}
380
381static inline bool bio_is_rw(struct bio *bio)
382{
383 if (!bio_has_data(bio))
384 return false;
385
386 if (bio->bi_rw & REQ_WRITE_SAME)
387 return false;
388
389 return true;
390}
391
392static inline bool bio_mergeable(struct bio *bio)
393{
394 if (bio->bi_rw & REQ_NOMERGE_FLAGS)
395 return false;
396
397 return true;
398}
399
400
401
402
403
404
405
406
407struct bio_list {
408 struct bio *head;
409 struct bio *tail;
410};
411
412static inline int bio_list_empty(const struct bio_list *bl)
413{
414 return bl->head == NULL;
415}
416
417static inline void bio_list_init(struct bio_list *bl)
418{
419 bl->head = bl->tail = NULL;
420}
421
422#define bio_list_for_each(bio, bl) \
423 for (bio = (bl)->head; bio; bio = bio->bi_next)
424
425static inline unsigned bio_list_size(const struct bio_list *bl)
426{
427 unsigned sz = 0;
428 struct bio *bio;
429
430 bio_list_for_each(bio, bl)
431 sz++;
432
433 return sz;
434}
435
436static inline void bio_list_add(struct bio_list *bl, struct bio *bio)
437{
438 bio->bi_next = NULL;
439
440 if (bl->tail)
441 bl->tail->bi_next = bio;
442 else
443 bl->head = bio;
444
445 bl->tail = bio;
446}
447
448static inline void bio_list_add_head(struct bio_list *bl, struct bio *bio)
449{
450 bio->bi_next = bl->head;
451
452 bl->head = bio;
453
454 if (!bl->tail)
455 bl->tail = bio;
456}
457
458static inline void bio_list_merge(struct bio_list *bl, struct bio_list *bl2)
459{
460 if (!bl2->head)
461 return;
462
463 if (bl->tail)
464 bl->tail->bi_next = bl2->head;
465 else
466 bl->head = bl2->head;
467
468 bl->tail = bl2->tail;
469}
470
471static inline void bio_list_merge_head(struct bio_list *bl,
472 struct bio_list *bl2)
473{
474 if (!bl2->head)
475 return;
476
477 if (bl->head)
478 bl2->tail->bi_next = bl->head;
479 else
480 bl->tail = bl2->tail;
481
482 bl->head = bl2->head;
483}
484
485static inline struct bio *bio_list_peek(struct bio_list *bl)
486{
487 return bl->head;
488}
489
490static inline struct bio *bio_list_pop(struct bio_list *bl)
491{
492 struct bio *bio = bl->head;
493
494 if (bio) {
495 bl->head = bl->head->bi_next;
496 if (!bl->head)
497 bl->tail = NULL;
498
499 bio->bi_next = NULL;
500 }
501
502 return bio;
503}
504
505static inline struct bio *bio_list_get(struct bio_list *bl)
506{
507 struct bio *bio = bl->head;
508
509 bl->head = bl->tail = NULL;
510
511 return bio;
512}
513
514
515
516
517
518
519
520#define BIO_POOL_SIZE 2
521#define BIOVEC_NR_POOLS 6
522#define BIOVEC_MAX_IDX (BIOVEC_NR_POOLS - 1)
523
524struct bio_set {
525 struct kmem_cache *bio_slab;
526 unsigned int front_pad;
527
528 mempool_t *bio_pool;
529 mempool_t *bvec_pool;
530#if defined(CONFIG_BLK_DEV_INTEGRITY)
531 mempool_t *bio_integrity_pool;
532 mempool_t *bvec_integrity_pool;
533#endif
534
535
536
537
538
539 spinlock_t rescue_lock;
540 struct bio_list rescue_list;
541 struct work_struct rescue_work;
542 struct workqueue_struct *rescue_workqueue;
543};
544
545struct biovec_slab {
546 int nr_vecs;
547 char *name;
548 struct kmem_cache *slab;
549};
550
551
552
553
554
555#define BIO_SPLIT_ENTRIES 2
556
557#if defined(CONFIG_BLK_DEV_INTEGRITY)
558
559#define bip_vec_idx(bip, idx) (&(bip->bip_vec[(idx)]))
560#define bip_vec(bip) bip_vec_idx(bip, 0)
561
562#define __bip_for_each_vec(bvl, bip, i, start_idx) \
563 for (bvl = bip_vec_idx((bip), (start_idx)), i = (start_idx); \
564 i < (bip)->bip_vcnt; \
565 bvl++, i++)
566
567#define bip_for_each_vec(bvl, bip, i) \
568 __bip_for_each_vec(bvl, bip, i, (bip)->bip_idx)
569
570#define bio_for_each_integrity_vec(_bvl, _bio, _iter) \
571 for_each_bio(_bio) \
572 bip_for_each_vec(_bvl, _bio->bi_integrity, _iter)
573
574#define bio_integrity(bio) (bio->bi_integrity != NULL)
575
576extern struct bio_integrity_payload *bio_integrity_alloc(struct bio *, gfp_t, unsigned int);
577extern void bio_integrity_free(struct bio *);
578extern int bio_integrity_add_page(struct bio *, struct page *, unsigned int, unsigned int);
579extern int bio_integrity_enabled(struct bio *bio);
580extern int bio_integrity_set_tag(struct bio *, void *, unsigned int);
581extern int bio_integrity_get_tag(struct bio *, void *, unsigned int);
582extern int bio_integrity_prep(struct bio *);
583extern void bio_integrity_endio(struct bio *, int);
584extern void bio_integrity_advance(struct bio *, unsigned int);
585extern void bio_integrity_trim(struct bio *, unsigned int, unsigned int);
586extern void bio_integrity_split(struct bio *, struct bio_pair *, int);
587extern int bio_integrity_clone(struct bio *, struct bio *, gfp_t);
588extern int bioset_integrity_create(struct bio_set *, int);
589extern void bioset_integrity_free(struct bio_set *);
590extern void bio_integrity_init(void);
591
592#else
593
594static inline int bio_integrity(struct bio *bio)
595{
596 return 0;
597}
598
599static inline int bio_integrity_enabled(struct bio *bio)
600{
601 return 0;
602}
603
604static inline int bioset_integrity_create(struct bio_set *bs, int pool_size)
605{
606 return 0;
607}
608
609static inline void bioset_integrity_free (struct bio_set *bs)
610{
611 return;
612}
613
614static inline int bio_integrity_prep(struct bio *bio)
615{
616 return 0;
617}
618
619static inline void bio_integrity_free(struct bio *bio)
620{
621 return;
622}
623
624static inline int bio_integrity_clone(struct bio *bio, struct bio *bio_src,
625 gfp_t gfp_mask)
626{
627 return 0;
628}
629
630static inline void bio_integrity_split(struct bio *bio, struct bio_pair *bp,
631 int sectors)
632{
633 return;
634}
635
636static inline void bio_integrity_advance(struct bio *bio,
637 unsigned int bytes_done)
638{
639 return;
640}
641
642static inline void bio_integrity_trim(struct bio *bio, unsigned int offset,
643 unsigned int sectors)
644{
645 return;
646}
647
648static inline void bio_integrity_init(void)
649{
650 return;
651}
652
653#endif
654
655#endif
656#endif
657