1#ifndef _LINUX_BLKDEV_H
2#define _LINUX_BLKDEV_H
3
4#include <linux/sched.h>
5
6#ifdef CONFIG_BLOCK
7
8#include <linux/major.h>
9#include <linux/genhd.h>
10#include <linux/list.h>
11#include <linux/llist.h>
12#include <linux/timer.h>
13#include <linux/workqueue.h>
14#include <linux/pagemap.h>
15#include <linux/backing-dev-defs.h>
16#include <linux/wait.h>
17#include <linux/mempool.h>
18#include <linux/pfn.h>
19#include <linux/bio.h>
20#include <linux/stringify.h>
21#include <linux/gfp.h>
22#include <linux/bsg.h>
23#include <linux/smp.h>
24#include <linux/rcupdate.h>
25#include <linux/percpu-refcount.h>
26#include <linux/scatterlist.h>
27
28struct module;
29struct scsi_ioctl_command;
30
31struct request_queue;
32struct elevator_queue;
33struct blk_trace;
34struct request;
35struct sg_io_hdr;
36struct bsg_job;
37struct blkcg_gq;
38struct blk_flush_queue;
39struct pr_ops;
40
41#define BLKDEV_MIN_RQ 4
42#define BLKDEV_MAX_RQ 128
43
44
45
46
47
48#define BLKCG_MAX_POLS 2
49
50struct request;
51typedef void (rq_end_io_fn)(struct request *, int);
52
53#define BLK_RL_SYNCFULL (1U << 0)
54#define BLK_RL_ASYNCFULL (1U << 1)
55
56struct request_list {
57 struct request_queue *q;
58#ifdef CONFIG_BLK_CGROUP
59 struct blkcg_gq *blkg;
60#endif
61
62
63
64
65 int count[2];
66 int starved[2];
67 mempool_t *rq_pool;
68 wait_queue_head_t wait[2];
69 unsigned int flags;
70};
71
72
73
74
75enum rq_cmd_type_bits {
76 REQ_TYPE_FS = 1,
77 REQ_TYPE_BLOCK_PC,
78 REQ_TYPE_DRV_PRIV,
79};
80
81#define BLK_MAX_CDB 16
82
83
84
85
86
87
88
89struct request {
90 struct list_head queuelist;
91 union {
92 struct call_single_data csd;
93 unsigned long fifo_time;
94 };
95
96 struct request_queue *q;
97 struct blk_mq_ctx *mq_ctx;
98
99 u64 cmd_flags;
100 unsigned cmd_type;
101 unsigned long atomic_flags;
102
103 int cpu;
104
105
106 unsigned int __data_len;
107 sector_t __sector;
108
109 struct bio *bio;
110 struct bio *biotail;
111
112
113
114
115
116
117
118
119 union {
120 struct hlist_node hash;
121 struct list_head ipi_list;
122 };
123
124
125
126
127
128
129 union {
130 struct rb_node rb_node;
131 void *completion_data;
132 };
133
134
135
136
137
138
139
140 union {
141 struct {
142 struct io_cq *icq;
143 void *priv[2];
144 } elv;
145
146 struct {
147 unsigned int seq;
148 struct list_head list;
149 rq_end_io_fn *saved_end_io;
150 } flush;
151 };
152
153 struct gendisk *rq_disk;
154 struct hd_struct *part;
155 unsigned long start_time;
156#ifdef CONFIG_BLK_CGROUP
157 struct request_list *rl;
158 unsigned long long start_time_ns;
159 unsigned long long io_start_time_ns;
160#endif
161
162
163
164 unsigned short nr_phys_segments;
165#if defined(CONFIG_BLK_DEV_INTEGRITY)
166 unsigned short nr_integrity_segments;
167#endif
168
169 unsigned short ioprio;
170
171 void *special;
172
173 int tag;
174 int errors;
175
176
177
178
179 unsigned char __cmd[BLK_MAX_CDB];
180 unsigned char *cmd;
181 unsigned short cmd_len;
182
183 unsigned int extra_len;
184 unsigned int sense_len;
185 unsigned int resid_len;
186 void *sense;
187
188 unsigned long deadline;
189 struct list_head timeout_list;
190 unsigned int timeout;
191 int retries;
192
193
194
195
196 rq_end_io_fn *end_io;
197 void *end_io_data;
198
199
200 struct request *next_rq;
201};
202
203static inline unsigned short req_get_ioprio(struct request *req)
204{
205 return req->ioprio;
206}
207
208#include <linux/elevator.h>
209
210struct blk_queue_ctx;
211
212typedef void (request_fn_proc) (struct request_queue *q);
213typedef blk_qc_t (make_request_fn) (struct request_queue *q, struct bio *bio);
214typedef int (prep_rq_fn) (struct request_queue *, struct request *);
215typedef void (unprep_rq_fn) (struct request_queue *, struct request *);
216
217struct bio_vec;
218typedef void (softirq_done_fn)(struct request *);
219typedef int (dma_drain_needed_fn)(struct request *);
220typedef int (lld_busy_fn) (struct request_queue *q);
221typedef int (bsg_job_fn) (struct bsg_job *);
222
223enum blk_eh_timer_return {
224 BLK_EH_NOT_HANDLED,
225 BLK_EH_HANDLED,
226 BLK_EH_RESET_TIMER,
227};
228
229typedef enum blk_eh_timer_return (rq_timed_out_fn)(struct request *);
230
231enum blk_queue_state {
232 Queue_down,
233 Queue_up,
234};
235
236struct blk_queue_tag {
237 struct request **tag_index;
238 unsigned long *tag_map;
239 int busy;
240 int max_depth;
241 int real_max_depth;
242 atomic_t refcnt;
243 int alloc_policy;
244 int next_tag;
245};
246#define BLK_TAG_ALLOC_FIFO 0
247#define BLK_TAG_ALLOC_RR 1
248
249#define BLK_SCSI_MAX_CMDS (256)
250#define BLK_SCSI_CMD_PER_LONG (BLK_SCSI_MAX_CMDS / (sizeof(long) * 8))
251
252struct queue_limits {
253 unsigned long bounce_pfn;
254 unsigned long seg_boundary_mask;
255 unsigned long virt_boundary_mask;
256
257 unsigned int max_hw_sectors;
258 unsigned int max_dev_sectors;
259 unsigned int chunk_sectors;
260 unsigned int max_sectors;
261 unsigned int max_segment_size;
262 unsigned int physical_block_size;
263 unsigned int alignment_offset;
264 unsigned int io_min;
265 unsigned int io_opt;
266 unsigned int max_discard_sectors;
267 unsigned int max_hw_discard_sectors;
268 unsigned int max_write_same_sectors;
269 unsigned int discard_granularity;
270 unsigned int discard_alignment;
271
272 unsigned short logical_block_size;
273 unsigned short max_segments;
274 unsigned short max_integrity_segments;
275
276 unsigned char misaligned;
277 unsigned char discard_misaligned;
278 unsigned char cluster;
279 unsigned char discard_zeroes_data;
280 unsigned char raid_partial_stripes_expensive;
281};
282
283struct request_queue {
284
285
286
287 struct list_head queue_head;
288 struct request *last_merge;
289 struct elevator_queue *elevator;
290 int nr_rqs[2];
291 int nr_rqs_elvpriv;
292
293
294
295
296
297
298
299 struct request_list root_rl;
300
301 request_fn_proc *request_fn;
302 make_request_fn *make_request_fn;
303 prep_rq_fn *prep_rq_fn;
304 unprep_rq_fn *unprep_rq_fn;
305 softirq_done_fn *softirq_done_fn;
306 rq_timed_out_fn *rq_timed_out_fn;
307 dma_drain_needed_fn *dma_drain_needed;
308 lld_busy_fn *lld_busy_fn;
309
310 struct blk_mq_ops *mq_ops;
311
312 unsigned int *mq_map;
313
314
315 struct blk_mq_ctx __percpu *queue_ctx;
316 unsigned int nr_queues;
317
318
319 struct blk_mq_hw_ctx **queue_hw_ctx;
320 unsigned int nr_hw_queues;
321
322
323
324
325 sector_t end_sector;
326 struct request *boundary_rq;
327
328
329
330
331 struct delayed_work delay_work;
332
333 struct backing_dev_info backing_dev_info;
334
335
336
337
338
339 void *queuedata;
340
341
342
343
344 unsigned long queue_flags;
345
346
347
348
349
350 int id;
351
352
353
354
355 gfp_t bounce_gfp;
356
357
358
359
360
361
362 spinlock_t __queue_lock;
363 spinlock_t *queue_lock;
364
365
366
367
368 struct kobject kobj;
369
370
371
372
373 struct kobject mq_kobj;
374
375#ifdef CONFIG_BLK_DEV_INTEGRITY
376 struct blk_integrity integrity;
377#endif
378
379#ifdef CONFIG_PM
380 struct device *dev;
381 int rpm_status;
382 unsigned int nr_pending;
383#endif
384
385
386
387
388 unsigned long nr_requests;
389 unsigned int nr_congestion_on;
390 unsigned int nr_congestion_off;
391 unsigned int nr_batching;
392
393 unsigned int dma_drain_size;
394 void *dma_drain_buffer;
395 unsigned int dma_pad_mask;
396 unsigned int dma_alignment;
397
398 struct blk_queue_tag *queue_tags;
399 struct list_head tag_busy_list;
400
401 unsigned int nr_sorted;
402 unsigned int in_flight[2];
403
404
405
406
407
408 unsigned int request_fn_active;
409
410 unsigned int rq_timeout;
411 struct timer_list timeout;
412 struct work_struct timeout_work;
413 struct list_head timeout_list;
414
415 struct list_head icq_list;
416#ifdef CONFIG_BLK_CGROUP
417 DECLARE_BITMAP (blkcg_pols, BLKCG_MAX_POLS);
418 struct blkcg_gq *root_blkg;
419 struct list_head blkg_list;
420#endif
421
422 struct queue_limits limits;
423
424
425
426
427 unsigned int sg_timeout;
428 unsigned int sg_reserved_size;
429 int node;
430#ifdef CONFIG_BLK_DEV_IO_TRACE
431 struct blk_trace *blk_trace;
432#endif
433
434
435
436 unsigned int flush_flags;
437 unsigned int flush_not_queueable:1;
438 struct blk_flush_queue *fq;
439
440 struct list_head requeue_list;
441 spinlock_t requeue_lock;
442 struct work_struct requeue_work;
443
444 struct mutex sysfs_lock;
445
446 int bypass_depth;
447 atomic_t mq_freeze_depth;
448
449#if defined(CONFIG_BLK_DEV_BSG)
450 bsg_job_fn *bsg_job_fn;
451 int bsg_job_size;
452 struct bsg_class_device bsg_dev;
453#endif
454
455#ifdef CONFIG_BLK_DEV_THROTTLING
456
457 struct throtl_data *td;
458#endif
459 struct rcu_head rcu_head;
460 wait_queue_head_t mq_freeze_wq;
461 struct percpu_ref q_usage_counter;
462 struct list_head all_q_node;
463
464 struct blk_mq_tag_set *tag_set;
465 struct list_head tag_set_list;
466 struct bio_set *bio_split;
467
468 bool mq_sysfs_init_done;
469};
470
471#define QUEUE_FLAG_QUEUED 1
472#define QUEUE_FLAG_STOPPED 2
473#define QUEUE_FLAG_SYNCFULL 3
474#define QUEUE_FLAG_ASYNCFULL 4
475#define QUEUE_FLAG_DYING 5
476#define QUEUE_FLAG_BYPASS 6
477#define QUEUE_FLAG_BIDI 7
478#define QUEUE_FLAG_NOMERGES 8
479#define QUEUE_FLAG_SAME_COMP 9
480#define QUEUE_FLAG_FAIL_IO 10
481#define QUEUE_FLAG_STACKABLE 11
482#define QUEUE_FLAG_NONROT 12
483#define QUEUE_FLAG_VIRT QUEUE_FLAG_NONROT
484#define QUEUE_FLAG_IO_STAT 13
485#define QUEUE_FLAG_DISCARD 14
486#define QUEUE_FLAG_NOXMERGES 15
487#define QUEUE_FLAG_ADD_RANDOM 16
488#define QUEUE_FLAG_SECDISCARD 17
489#define QUEUE_FLAG_SAME_FORCE 18
490#define QUEUE_FLAG_DEAD 19
491#define QUEUE_FLAG_INIT_DONE 20
492#define QUEUE_FLAG_NO_SG_MERGE 21
493#define QUEUE_FLAG_POLL 22
494
495#define QUEUE_FLAG_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \
496 (1 << QUEUE_FLAG_STACKABLE) | \
497 (1 << QUEUE_FLAG_SAME_COMP) | \
498 (1 << QUEUE_FLAG_ADD_RANDOM))
499
500#define QUEUE_FLAG_MQ_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \
501 (1 << QUEUE_FLAG_STACKABLE) | \
502 (1 << QUEUE_FLAG_SAME_COMP) | \
503 (1 << QUEUE_FLAG_POLL))
504
505static inline void queue_lockdep_assert_held(struct request_queue *q)
506{
507 if (q->queue_lock)
508 lockdep_assert_held(q->queue_lock);
509}
510
511static inline void queue_flag_set_unlocked(unsigned int flag,
512 struct request_queue *q)
513{
514 __set_bit(flag, &q->queue_flags);
515}
516
517static inline int queue_flag_test_and_clear(unsigned int flag,
518 struct request_queue *q)
519{
520 queue_lockdep_assert_held(q);
521
522 if (test_bit(flag, &q->queue_flags)) {
523 __clear_bit(flag, &q->queue_flags);
524 return 1;
525 }
526
527 return 0;
528}
529
530static inline int queue_flag_test_and_set(unsigned int flag,
531 struct request_queue *q)
532{
533 queue_lockdep_assert_held(q);
534
535 if (!test_bit(flag, &q->queue_flags)) {
536 __set_bit(flag, &q->queue_flags);
537 return 0;
538 }
539
540 return 1;
541}
542
543static inline void queue_flag_set(unsigned int flag, struct request_queue *q)
544{
545 queue_lockdep_assert_held(q);
546 __set_bit(flag, &q->queue_flags);
547}
548
549static inline void queue_flag_clear_unlocked(unsigned int flag,
550 struct request_queue *q)
551{
552 __clear_bit(flag, &q->queue_flags);
553}
554
555static inline int queue_in_flight(struct request_queue *q)
556{
557 return q->in_flight[0] + q->in_flight[1];
558}
559
560static inline void queue_flag_clear(unsigned int flag, struct request_queue *q)
561{
562 queue_lockdep_assert_held(q);
563 __clear_bit(flag, &q->queue_flags);
564}
565
566#define blk_queue_tagged(q) test_bit(QUEUE_FLAG_QUEUED, &(q)->queue_flags)
567#define blk_queue_stopped(q) test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags)
568#define blk_queue_dying(q) test_bit(QUEUE_FLAG_DYING, &(q)->queue_flags)
569#define blk_queue_dead(q) test_bit(QUEUE_FLAG_DEAD, &(q)->queue_flags)
570#define blk_queue_bypass(q) test_bit(QUEUE_FLAG_BYPASS, &(q)->queue_flags)
571#define blk_queue_init_done(q) test_bit(QUEUE_FLAG_INIT_DONE, &(q)->queue_flags)
572#define blk_queue_nomerges(q) test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags)
573#define blk_queue_noxmerges(q) \
574 test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags)
575#define blk_queue_nonrot(q) test_bit(QUEUE_FLAG_NONROT, &(q)->queue_flags)
576#define blk_queue_io_stat(q) test_bit(QUEUE_FLAG_IO_STAT, &(q)->queue_flags)
577#define blk_queue_add_random(q) test_bit(QUEUE_FLAG_ADD_RANDOM, &(q)->queue_flags)
578#define blk_queue_stackable(q) \
579 test_bit(QUEUE_FLAG_STACKABLE, &(q)->queue_flags)
580#define blk_queue_discard(q) test_bit(QUEUE_FLAG_DISCARD, &(q)->queue_flags)
581#define blk_queue_secdiscard(q) (blk_queue_discard(q) && \
582 test_bit(QUEUE_FLAG_SECDISCARD, &(q)->queue_flags))
583
584#define blk_noretry_request(rq) \
585 ((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \
586 REQ_FAILFAST_DRIVER))
587
588#define blk_account_rq(rq) \
589 (((rq)->cmd_flags & REQ_STARTED) && \
590 ((rq)->cmd_type == REQ_TYPE_FS))
591
592#define blk_rq_cpu_valid(rq) ((rq)->cpu != -1)
593#define blk_bidi_rq(rq) ((rq)->next_rq != NULL)
594
595#define blk_queued_rq(rq) (!list_empty(&(rq)->queuelist))
596
597#define list_entry_rq(ptr) list_entry((ptr), struct request, queuelist)
598
599#define rq_data_dir(rq) ((int)((rq)->cmd_flags & 1))
600
601
602
603
604
605static inline bool queue_is_rq_based(struct request_queue *q)
606{
607 return q->request_fn || q->mq_ops;
608}
609
610static inline unsigned int blk_queue_cluster(struct request_queue *q)
611{
612 return q->limits.cluster;
613}
614
615
616
617
618static inline bool rw_is_sync(unsigned int rw_flags)
619{
620 return !(rw_flags & REQ_WRITE) || (rw_flags & REQ_SYNC);
621}
622
623static inline bool rq_is_sync(struct request *rq)
624{
625 return rw_is_sync(rq->cmd_flags);
626}
627
628static inline bool blk_rl_full(struct request_list *rl, bool sync)
629{
630 unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL;
631
632 return rl->flags & flag;
633}
634
635static inline void blk_set_rl_full(struct request_list *rl, bool sync)
636{
637 unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL;
638
639 rl->flags |= flag;
640}
641
642static inline void blk_clear_rl_full(struct request_list *rl, bool sync)
643{
644 unsigned int flag = sync ? BLK_RL_SYNCFULL : BLK_RL_ASYNCFULL;
645
646 rl->flags &= ~flag;
647}
648
649static inline bool rq_mergeable(struct request *rq)
650{
651 if (rq->cmd_type != REQ_TYPE_FS)
652 return false;
653
654 if (rq->cmd_flags & REQ_NOMERGE_FLAGS)
655 return false;
656
657 return true;
658}
659
660static inline bool blk_check_merge_flags(unsigned int flags1,
661 unsigned int flags2)
662{
663 if ((flags1 & REQ_DISCARD) != (flags2 & REQ_DISCARD))
664 return false;
665
666 if ((flags1 & REQ_SECURE) != (flags2 & REQ_SECURE))
667 return false;
668
669 if ((flags1 & REQ_WRITE_SAME) != (flags2 & REQ_WRITE_SAME))
670 return false;
671
672 return true;
673}
674
675static inline bool blk_write_same_mergeable(struct bio *a, struct bio *b)
676{
677 if (bio_data(a) == bio_data(b))
678 return true;
679
680 return false;
681}
682
683
684
685
686enum {
687 BLKPREP_OK,
688 BLKPREP_KILL,
689 BLKPREP_DEFER,
690 BLKPREP_INVALID,
691};
692
693extern unsigned long blk_max_low_pfn, blk_max_pfn;
694
695
696
697
698
699
700
701
702
703#if BITS_PER_LONG == 32
704#define BLK_BOUNCE_HIGH ((u64)blk_max_low_pfn << PAGE_SHIFT)
705#else
706#define BLK_BOUNCE_HIGH -1ULL
707#endif
708#define BLK_BOUNCE_ANY (-1ULL)
709#define BLK_BOUNCE_ISA (DMA_BIT_MASK(24))
710
711
712
713
714#define BLK_DEFAULT_SG_TIMEOUT (60 * HZ)
715#define BLK_MIN_SG_TIMEOUT (7 * HZ)
716
717#ifdef CONFIG_BOUNCE
718extern int init_emergency_isa_pool(void);
719extern void blk_queue_bounce(struct request_queue *q, struct bio **bio);
720#else
721static inline int init_emergency_isa_pool(void)
722{
723 return 0;
724}
725static inline void blk_queue_bounce(struct request_queue *q, struct bio **bio)
726{
727}
728#endif
729
730struct rq_map_data {
731 struct page **pages;
732 int page_order;
733 int nr_entries;
734 unsigned long offset;
735 int null_mapped;
736 int from_user;
737};
738
739struct req_iterator {
740 struct bvec_iter iter;
741 struct bio *bio;
742};
743
744
745#define for_each_bio(_bio) \
746 for (; _bio; _bio = _bio->bi_next)
747#define __rq_for_each_bio(_bio, rq) \
748 if ((rq->bio)) \
749 for (_bio = (rq)->bio; _bio; _bio = _bio->bi_next)
750
751#define rq_for_each_segment(bvl, _rq, _iter) \
752 __rq_for_each_bio(_iter.bio, _rq) \
753 bio_for_each_segment(bvl, _iter.bio, _iter.iter)
754
755#define rq_iter_last(bvec, _iter) \
756 (_iter.bio->bi_next == NULL && \
757 bio_iter_last(bvec, _iter.iter))
758
759#ifndef ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
760# error "You should define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE for your platform"
761#endif
762#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
763extern void rq_flush_dcache_pages(struct request *rq);
764#else
765static inline void rq_flush_dcache_pages(struct request *rq)
766{
767}
768#endif
769
770extern int blk_register_queue(struct gendisk *disk);
771extern void blk_unregister_queue(struct gendisk *disk);
772extern blk_qc_t generic_make_request(struct bio *bio);
773extern void blk_rq_init(struct request_queue *q, struct request *rq);
774extern void blk_put_request(struct request *);
775extern void __blk_put_request(struct request_queue *, struct request *);
776extern struct request *blk_get_request(struct request_queue *, int, gfp_t);
777extern struct request *blk_make_request(struct request_queue *, struct bio *,
778 gfp_t);
779extern void blk_rq_set_block_pc(struct request *);
780extern void blk_requeue_request(struct request_queue *, struct request *);
781extern void blk_add_request_payload(struct request *rq, struct page *page,
782 unsigned int len);
783extern int blk_lld_busy(struct request_queue *q);
784extern int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
785 struct bio_set *bs, gfp_t gfp_mask,
786 int (*bio_ctr)(struct bio *, struct bio *, void *),
787 void *data);
788extern void blk_rq_unprep_clone(struct request *rq);
789extern int blk_insert_cloned_request(struct request_queue *q,
790 struct request *rq);
791extern void blk_delay_queue(struct request_queue *, unsigned long);
792extern void blk_queue_split(struct request_queue *, struct bio **,
793 struct bio_set *);
794extern void blk_recount_segments(struct request_queue *, struct bio *);
795extern int scsi_verify_blk_ioctl(struct block_device *, unsigned int);
796extern int scsi_cmd_blk_ioctl(struct block_device *, fmode_t,
797 unsigned int, void __user *);
798extern int scsi_cmd_ioctl(struct request_queue *, struct gendisk *, fmode_t,
799 unsigned int, void __user *);
800extern int sg_scsi_ioctl(struct request_queue *, struct gendisk *, fmode_t,
801 struct scsi_ioctl_command __user *);
802
803extern int blk_queue_enter(struct request_queue *q, bool nowait);
804extern void blk_queue_exit(struct request_queue *q);
805extern void blk_start_queue(struct request_queue *q);
806extern void blk_start_queue_async(struct request_queue *q);
807extern void blk_stop_queue(struct request_queue *q);
808extern void blk_sync_queue(struct request_queue *q);
809extern void __blk_stop_queue(struct request_queue *q);
810extern void __blk_run_queue(struct request_queue *q);
811extern void __blk_run_queue_uncond(struct request_queue *q);
812extern void blk_run_queue(struct request_queue *);
813extern void blk_run_queue_async(struct request_queue *q);
814extern int blk_rq_map_user(struct request_queue *, struct request *,
815 struct rq_map_data *, void __user *, unsigned long,
816 gfp_t);
817extern int blk_rq_unmap_user(struct bio *);
818extern int blk_rq_map_kern(struct request_queue *, struct request *, void *, unsigned int, gfp_t);
819extern int blk_rq_map_user_iov(struct request_queue *, struct request *,
820 struct rq_map_data *, const struct iov_iter *,
821 gfp_t);
822extern int blk_execute_rq(struct request_queue *, struct gendisk *,
823 struct request *, int);
824extern void blk_execute_rq_nowait(struct request_queue *, struct gendisk *,
825 struct request *, int, rq_end_io_fn *);
826
827bool blk_poll(struct request_queue *q, blk_qc_t cookie);
828
829static inline struct request_queue *bdev_get_queue(struct block_device *bdev)
830{
831 return bdev->bd_disk->queue;
832}
833
834
835
836
837
838
839
840
841
842static inline sector_t blk_rq_pos(const struct request *rq)
843{
844 return rq->__sector;
845}
846
847static inline unsigned int blk_rq_bytes(const struct request *rq)
848{
849 return rq->__data_len;
850}
851
852static inline int blk_rq_cur_bytes(const struct request *rq)
853{
854 return rq->bio ? bio_cur_bytes(rq->bio) : 0;
855}
856
857extern unsigned int blk_rq_err_bytes(const struct request *rq);
858
859static inline unsigned int blk_rq_sectors(const struct request *rq)
860{
861 return blk_rq_bytes(rq) >> 9;
862}
863
864static inline unsigned int blk_rq_cur_sectors(const struct request *rq)
865{
866 return blk_rq_cur_bytes(rq) >> 9;
867}
868
869static inline unsigned int blk_queue_get_max_sectors(struct request_queue *q,
870 unsigned int cmd_flags)
871{
872 if (unlikely(cmd_flags & REQ_DISCARD))
873 return min(q->limits.max_discard_sectors, UINT_MAX >> 9);
874
875 if (unlikely(cmd_flags & REQ_WRITE_SAME))
876 return q->limits.max_write_same_sectors;
877
878 return q->limits.max_sectors;
879}
880
881
882
883
884
885static inline unsigned int blk_max_size_offset(struct request_queue *q,
886 sector_t offset)
887{
888 if (!q->limits.chunk_sectors)
889 return q->limits.max_sectors;
890
891 return q->limits.chunk_sectors -
892 (offset & (q->limits.chunk_sectors - 1));
893}
894
895static inline unsigned int blk_rq_get_max_sectors(struct request *rq)
896{
897 struct request_queue *q = rq->q;
898
899 if (unlikely(rq->cmd_type != REQ_TYPE_FS))
900 return q->limits.max_hw_sectors;
901
902 if (!q->limits.chunk_sectors || (rq->cmd_flags & REQ_DISCARD))
903 return blk_queue_get_max_sectors(q, rq->cmd_flags);
904
905 return min(blk_max_size_offset(q, blk_rq_pos(rq)),
906 blk_queue_get_max_sectors(q, rq->cmd_flags));
907}
908
909static inline unsigned int blk_rq_count_bios(struct request *rq)
910{
911 unsigned int nr_bios = 0;
912 struct bio *bio;
913
914 __rq_for_each_bio(bio, rq)
915 nr_bios++;
916
917 return nr_bios;
918}
919
920
921
922
923extern struct request *blk_peek_request(struct request_queue *q);
924extern void blk_start_request(struct request *rq);
925extern struct request *blk_fetch_request(struct request_queue *q);
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940extern bool blk_update_request(struct request *rq, int error,
941 unsigned int nr_bytes);
942extern void blk_finish_request(struct request *rq, int error);
943extern bool blk_end_request(struct request *rq, int error,
944 unsigned int nr_bytes);
945extern void blk_end_request_all(struct request *rq, int error);
946extern bool blk_end_request_cur(struct request *rq, int error);
947extern bool blk_end_request_err(struct request *rq, int error);
948extern bool __blk_end_request(struct request *rq, int error,
949 unsigned int nr_bytes);
950extern void __blk_end_request_all(struct request *rq, int error);
951extern bool __blk_end_request_cur(struct request *rq, int error);
952extern bool __blk_end_request_err(struct request *rq, int error);
953
954extern void blk_complete_request(struct request *);
955extern void __blk_complete_request(struct request *);
956extern void blk_abort_request(struct request *);
957extern void blk_unprep_request(struct request *);
958
959
960
961
962extern struct request_queue *blk_init_queue_node(request_fn_proc *rfn,
963 spinlock_t *lock, int node_id);
964extern struct request_queue *blk_init_queue(request_fn_proc *, spinlock_t *);
965extern struct request_queue *blk_init_allocated_queue(struct request_queue *,
966 request_fn_proc *, spinlock_t *);
967extern void blk_cleanup_queue(struct request_queue *);
968extern void blk_queue_make_request(struct request_queue *, make_request_fn *);
969extern void blk_queue_bounce_limit(struct request_queue *, u64);
970extern void blk_queue_max_hw_sectors(struct request_queue *, unsigned int);
971extern void blk_queue_chunk_sectors(struct request_queue *, unsigned int);
972extern void blk_queue_max_segments(struct request_queue *, unsigned short);
973extern void blk_queue_max_segment_size(struct request_queue *, unsigned int);
974extern void blk_queue_max_discard_sectors(struct request_queue *q,
975 unsigned int max_discard_sectors);
976extern void blk_queue_max_write_same_sectors(struct request_queue *q,
977 unsigned int max_write_same_sectors);
978extern void blk_queue_logical_block_size(struct request_queue *, unsigned short);
979extern void blk_queue_physical_block_size(struct request_queue *, unsigned int);
980extern void blk_queue_alignment_offset(struct request_queue *q,
981 unsigned int alignment);
982extern void blk_limits_io_min(struct queue_limits *limits, unsigned int min);
983extern void blk_queue_io_min(struct request_queue *q, unsigned int min);
984extern void blk_limits_io_opt(struct queue_limits *limits, unsigned int opt);
985extern void blk_queue_io_opt(struct request_queue *q, unsigned int opt);
986extern void blk_set_default_limits(struct queue_limits *lim);
987extern void blk_set_stacking_limits(struct queue_limits *lim);
988extern int blk_stack_limits(struct queue_limits *t, struct queue_limits *b,
989 sector_t offset);
990extern int bdev_stack_limits(struct queue_limits *t, struct block_device *bdev,
991 sector_t offset);
992extern void disk_stack_limits(struct gendisk *disk, struct block_device *bdev,
993 sector_t offset);
994extern void blk_queue_stack_limits(struct request_queue *t, struct request_queue *b);
995extern void blk_queue_dma_pad(struct request_queue *, unsigned int);
996extern void blk_queue_update_dma_pad(struct request_queue *, unsigned int);
997extern int blk_queue_dma_drain(struct request_queue *q,
998 dma_drain_needed_fn *dma_drain_needed,
999 void *buf, unsigned int size);
1000extern void blk_queue_lld_busy(struct request_queue *q, lld_busy_fn *fn);
1001extern void blk_queue_segment_boundary(struct request_queue *, unsigned long);
1002extern void blk_queue_virt_boundary(struct request_queue *, unsigned long);
1003extern void blk_queue_prep_rq(struct request_queue *, prep_rq_fn *pfn);
1004extern void blk_queue_unprep_rq(struct request_queue *, unprep_rq_fn *ufn);
1005extern void blk_queue_dma_alignment(struct request_queue *, int);
1006extern void blk_queue_update_dma_alignment(struct request_queue *, int);
1007extern void blk_queue_softirq_done(struct request_queue *, softirq_done_fn *);
1008extern void blk_queue_rq_timed_out(struct request_queue *, rq_timed_out_fn *);
1009extern void blk_queue_rq_timeout(struct request_queue *, unsigned int);
1010extern void blk_queue_flush(struct request_queue *q, unsigned int flush);
1011extern void blk_queue_flush_queueable(struct request_queue *q, bool queueable);
1012extern struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev);
1013
1014extern int blk_rq_map_sg(struct request_queue *, struct request *, struct scatterlist *);
1015extern void blk_dump_rq_flags(struct request *, char *);
1016extern long nr_blockdev_pages(void);
1017
1018bool __must_check blk_get_queue(struct request_queue *);
1019struct request_queue *blk_alloc_queue(gfp_t);
1020struct request_queue *blk_alloc_queue_node(gfp_t, int);
1021extern void blk_put_queue(struct request_queue *);
1022extern void blk_set_queue_dying(struct request_queue *);
1023
1024
1025
1026
1027#ifdef CONFIG_PM
1028extern void blk_pm_runtime_init(struct request_queue *q, struct device *dev);
1029extern int blk_pre_runtime_suspend(struct request_queue *q);
1030extern void blk_post_runtime_suspend(struct request_queue *q, int err);
1031extern void blk_pre_runtime_resume(struct request_queue *q);
1032extern void blk_post_runtime_resume(struct request_queue *q, int err);
1033extern void blk_set_runtime_active(struct request_queue *q);
1034#else
1035static inline void blk_pm_runtime_init(struct request_queue *q,
1036 struct device *dev) {}
1037static inline int blk_pre_runtime_suspend(struct request_queue *q)
1038{
1039 return -ENOSYS;
1040}
1041static inline void blk_post_runtime_suspend(struct request_queue *q, int err) {}
1042static inline void blk_pre_runtime_resume(struct request_queue *q) {}
1043static inline void blk_post_runtime_resume(struct request_queue *q, int err) {}
1044extern inline void blk_set_runtime_active(struct request_queue *q) {}
1045#endif
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059struct blk_plug {
1060 struct list_head list;
1061 struct list_head mq_list;
1062 struct list_head cb_list;
1063};
1064#define BLK_MAX_REQUEST_COUNT 16
1065
1066struct blk_plug_cb;
1067typedef void (*blk_plug_cb_fn)(struct blk_plug_cb *, bool);
1068struct blk_plug_cb {
1069 struct list_head list;
1070 blk_plug_cb_fn callback;
1071 void *data;
1072};
1073extern struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug,
1074 void *data, int size);
1075extern void blk_start_plug(struct blk_plug *);
1076extern void blk_finish_plug(struct blk_plug *);
1077extern void blk_flush_plug_list(struct blk_plug *, bool);
1078
1079static inline void blk_flush_plug(struct task_struct *tsk)
1080{
1081 struct blk_plug *plug = tsk->plug;
1082
1083 if (plug)
1084 blk_flush_plug_list(plug, false);
1085}
1086
1087static inline void blk_schedule_flush_plug(struct task_struct *tsk)
1088{
1089 struct blk_plug *plug = tsk->plug;
1090
1091 if (plug)
1092 blk_flush_plug_list(plug, true);
1093}
1094
1095static inline bool blk_needs_flush_plug(struct task_struct *tsk)
1096{
1097 struct blk_plug *plug = tsk->plug;
1098
1099 return plug &&
1100 (!list_empty(&plug->list) ||
1101 !list_empty(&plug->mq_list) ||
1102 !list_empty(&plug->cb_list));
1103}
1104
1105
1106
1107
1108extern int blk_queue_start_tag(struct request_queue *, struct request *);
1109extern struct request *blk_queue_find_tag(struct request_queue *, int);
1110extern void blk_queue_end_tag(struct request_queue *, struct request *);
1111extern int blk_queue_init_tags(struct request_queue *, int, struct blk_queue_tag *, int);
1112extern void blk_queue_free_tags(struct request_queue *);
1113extern int blk_queue_resize_tags(struct request_queue *, int);
1114extern void blk_queue_invalidate_tags(struct request_queue *);
1115extern struct blk_queue_tag *blk_init_tags(int, int);
1116extern void blk_free_tags(struct blk_queue_tag *);
1117
1118static inline struct request *blk_map_queue_find_tag(struct blk_queue_tag *bqt,
1119 int tag)
1120{
1121 if (unlikely(bqt == NULL || tag >= bqt->real_max_depth))
1122 return NULL;
1123 return bqt->tag_index[tag];
1124}
1125
1126#define BLKDEV_DISCARD_SECURE 0x01
1127
1128extern int blkdev_issue_flush(struct block_device *, gfp_t, sector_t *);
1129extern int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
1130 sector_t nr_sects, gfp_t gfp_mask, unsigned long flags);
1131extern int blkdev_issue_write_same(struct block_device *bdev, sector_t sector,
1132 sector_t nr_sects, gfp_t gfp_mask, struct page *page);
1133extern int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
1134 sector_t nr_sects, gfp_t gfp_mask, bool discard);
1135static inline int sb_issue_discard(struct super_block *sb, sector_t block,
1136 sector_t nr_blocks, gfp_t gfp_mask, unsigned long flags)
1137{
1138 return blkdev_issue_discard(sb->s_bdev, block << (sb->s_blocksize_bits - 9),
1139 nr_blocks << (sb->s_blocksize_bits - 9),
1140 gfp_mask, flags);
1141}
1142static inline int sb_issue_zeroout(struct super_block *sb, sector_t block,
1143 sector_t nr_blocks, gfp_t gfp_mask)
1144{
1145 return blkdev_issue_zeroout(sb->s_bdev,
1146 block << (sb->s_blocksize_bits - 9),
1147 nr_blocks << (sb->s_blocksize_bits - 9),
1148 gfp_mask, true);
1149}
1150
1151extern int blk_verify_command(unsigned char *cmd, fmode_t has_write_perm);
1152
1153enum blk_default_limits {
1154 BLK_MAX_SEGMENTS = 128,
1155 BLK_SAFE_MAX_SECTORS = 255,
1156 BLK_DEF_MAX_SECTORS = 2560,
1157 BLK_MAX_SEGMENT_SIZE = 65536,
1158 BLK_SEG_BOUNDARY_MASK = 0xFFFFFFFFUL,
1159};
1160
1161#define blkdev_entry_to_request(entry) list_entry((entry), struct request, queuelist)
1162
1163static inline unsigned long queue_bounce_pfn(struct request_queue *q)
1164{
1165 return q->limits.bounce_pfn;
1166}
1167
1168static inline unsigned long queue_segment_boundary(struct request_queue *q)
1169{
1170 return q->limits.seg_boundary_mask;
1171}
1172
1173static inline unsigned long queue_virt_boundary(struct request_queue *q)
1174{
1175 return q->limits.virt_boundary_mask;
1176}
1177
1178static inline unsigned int queue_max_sectors(struct request_queue *q)
1179{
1180 return q->limits.max_sectors;
1181}
1182
1183static inline unsigned int queue_max_hw_sectors(struct request_queue *q)
1184{
1185 return q->limits.max_hw_sectors;
1186}
1187
1188static inline unsigned short queue_max_segments(struct request_queue *q)
1189{
1190 return q->limits.max_segments;
1191}
1192
1193static inline unsigned int queue_max_segment_size(struct request_queue *q)
1194{
1195 return q->limits.max_segment_size;
1196}
1197
1198static inline unsigned short queue_logical_block_size(struct request_queue *q)
1199{
1200 int retval = 512;
1201
1202 if (q && q->limits.logical_block_size)
1203 retval = q->limits.logical_block_size;
1204
1205 return retval;
1206}
1207
1208static inline unsigned short bdev_logical_block_size(struct block_device *bdev)
1209{
1210 return queue_logical_block_size(bdev_get_queue(bdev));
1211}
1212
1213static inline unsigned int queue_physical_block_size(struct request_queue *q)
1214{
1215 return q->limits.physical_block_size;
1216}
1217
1218static inline unsigned int bdev_physical_block_size(struct block_device *bdev)
1219{
1220 return queue_physical_block_size(bdev_get_queue(bdev));
1221}
1222
1223static inline unsigned int queue_io_min(struct request_queue *q)
1224{
1225 return q->limits.io_min;
1226}
1227
1228static inline int bdev_io_min(struct block_device *bdev)
1229{
1230 return queue_io_min(bdev_get_queue(bdev));
1231}
1232
1233static inline unsigned int queue_io_opt(struct request_queue *q)
1234{
1235 return q->limits.io_opt;
1236}
1237
1238static inline int bdev_io_opt(struct block_device *bdev)
1239{
1240 return queue_io_opt(bdev_get_queue(bdev));
1241}
1242
1243static inline int queue_alignment_offset(struct request_queue *q)
1244{
1245 if (q->limits.misaligned)
1246 return -1;
1247
1248 return q->limits.alignment_offset;
1249}
1250
1251static inline int queue_limit_alignment_offset(struct queue_limits *lim, sector_t sector)
1252{
1253 unsigned int granularity = max(lim->physical_block_size, lim->io_min);
1254 unsigned int alignment = sector_div(sector, granularity >> 9) << 9;
1255
1256 return (granularity + lim->alignment_offset - alignment) % granularity;
1257}
1258
1259static inline int bdev_alignment_offset(struct block_device *bdev)
1260{
1261 struct request_queue *q = bdev_get_queue(bdev);
1262
1263 if (q->limits.misaligned)
1264 return -1;
1265
1266 if (bdev != bdev->bd_contains)
1267 return bdev->bd_part->alignment_offset;
1268
1269 return q->limits.alignment_offset;
1270}
1271
1272static inline int queue_discard_alignment(struct request_queue *q)
1273{
1274 if (q->limits.discard_misaligned)
1275 return -1;
1276
1277 return q->limits.discard_alignment;
1278}
1279
1280static inline int queue_limit_discard_alignment(struct queue_limits *lim, sector_t sector)
1281{
1282 unsigned int alignment, granularity, offset;
1283
1284 if (!lim->max_discard_sectors)
1285 return 0;
1286
1287
1288 alignment = lim->discard_alignment >> 9;
1289 granularity = lim->discard_granularity >> 9;
1290 if (!granularity)
1291 return 0;
1292
1293
1294 offset = sector_div(sector, granularity);
1295
1296
1297 offset = (granularity + alignment - offset) % granularity;
1298
1299
1300 return offset << 9;
1301}
1302
1303static inline int bdev_discard_alignment(struct block_device *bdev)
1304{
1305 struct request_queue *q = bdev_get_queue(bdev);
1306
1307 if (bdev != bdev->bd_contains)
1308 return bdev->bd_part->discard_alignment;
1309
1310 return q->limits.discard_alignment;
1311}
1312
1313static inline unsigned int queue_discard_zeroes_data(struct request_queue *q)
1314{
1315 if (q->limits.max_discard_sectors && q->limits.discard_zeroes_data == 1)
1316 return 1;
1317
1318 return 0;
1319}
1320
1321static inline unsigned int bdev_discard_zeroes_data(struct block_device *bdev)
1322{
1323 return queue_discard_zeroes_data(bdev_get_queue(bdev));
1324}
1325
1326static inline unsigned int bdev_write_same(struct block_device *bdev)
1327{
1328 struct request_queue *q = bdev_get_queue(bdev);
1329
1330 if (q)
1331 return q->limits.max_write_same_sectors;
1332
1333 return 0;
1334}
1335
1336static inline int queue_dma_alignment(struct request_queue *q)
1337{
1338 return q ? q->dma_alignment : 511;
1339}
1340
1341static inline int blk_rq_aligned(struct request_queue *q, unsigned long addr,
1342 unsigned int len)
1343{
1344 unsigned int alignment = queue_dma_alignment(q) | q->dma_pad_mask;
1345 return !(addr & alignment) && !(len & alignment);
1346}
1347
1348
1349static inline unsigned int blksize_bits(unsigned int size)
1350{
1351 unsigned int bits = 8;
1352 do {
1353 bits++;
1354 size >>= 1;
1355 } while (size > 256);
1356 return bits;
1357}
1358
1359static inline unsigned int block_size(struct block_device *bdev)
1360{
1361 return bdev->bd_block_size;
1362}
1363
1364static inline bool queue_flush_queueable(struct request_queue *q)
1365{
1366 return !q->flush_not_queueable;
1367}
1368
1369typedef struct {struct page *v;} Sector;
1370
1371unsigned char *read_dev_sector(struct block_device *, sector_t, Sector *);
1372
1373static inline void put_dev_sector(Sector p)
1374{
1375 put_page(p.v);
1376}
1377
1378static inline bool __bvec_gap_to_prev(struct request_queue *q,
1379 struct bio_vec *bprv, unsigned int offset)
1380{
1381 return offset ||
1382 ((bprv->bv_offset + bprv->bv_len) & queue_virt_boundary(q));
1383}
1384
1385
1386
1387
1388
1389static inline bool bvec_gap_to_prev(struct request_queue *q,
1390 struct bio_vec *bprv, unsigned int offset)
1391{
1392 if (!queue_virt_boundary(q))
1393 return false;
1394 return __bvec_gap_to_prev(q, bprv, offset);
1395}
1396
1397static inline bool bio_will_gap(struct request_queue *q, struct bio *prev,
1398 struct bio *next)
1399{
1400 if (bio_has_data(prev) && queue_virt_boundary(q)) {
1401 struct bio_vec pb, nb;
1402
1403 bio_get_last_bvec(prev, &pb);
1404 bio_get_first_bvec(next, &nb);
1405
1406 return __bvec_gap_to_prev(q, &pb, nb.bv_offset);
1407 }
1408
1409 return false;
1410}
1411
1412static inline bool req_gap_back_merge(struct request *req, struct bio *bio)
1413{
1414 return bio_will_gap(req->q, req->biotail, bio);
1415}
1416
1417static inline bool req_gap_front_merge(struct request *req, struct bio *bio)
1418{
1419 return bio_will_gap(req->q, bio, req->bio);
1420}
1421
1422struct work_struct;
1423int kblockd_schedule_work(struct work_struct *work);
1424int kblockd_schedule_delayed_work(struct delayed_work *dwork, unsigned long delay);
1425int kblockd_schedule_delayed_work_on(int cpu, struct delayed_work *dwork, unsigned long delay);
1426
1427#ifdef CONFIG_BLK_CGROUP
1428
1429
1430
1431
1432
1433static inline void set_start_time_ns(struct request *req)
1434{
1435 preempt_disable();
1436 req->start_time_ns = sched_clock();
1437 preempt_enable();
1438}
1439
1440static inline void set_io_start_time_ns(struct request *req)
1441{
1442 preempt_disable();
1443 req->io_start_time_ns = sched_clock();
1444 preempt_enable();
1445}
1446
1447static inline uint64_t rq_start_time_ns(struct request *req)
1448{
1449 return req->start_time_ns;
1450}
1451
1452static inline uint64_t rq_io_start_time_ns(struct request *req)
1453{
1454 return req->io_start_time_ns;
1455}
1456#else
1457static inline void set_start_time_ns(struct request *req) {}
1458static inline void set_io_start_time_ns(struct request *req) {}
1459static inline uint64_t rq_start_time_ns(struct request *req)
1460{
1461 return 0;
1462}
1463static inline uint64_t rq_io_start_time_ns(struct request *req)
1464{
1465 return 0;
1466}
1467#endif
1468
1469#define MODULE_ALIAS_BLOCKDEV(major,minor) \
1470 MODULE_ALIAS("block-major-" __stringify(major) "-" __stringify(minor))
1471#define MODULE_ALIAS_BLOCKDEV_MAJOR(major) \
1472 MODULE_ALIAS("block-major-" __stringify(major) "-*")
1473
1474#if defined(CONFIG_BLK_DEV_INTEGRITY)
1475
1476enum blk_integrity_flags {
1477 BLK_INTEGRITY_VERIFY = 1 << 0,
1478 BLK_INTEGRITY_GENERATE = 1 << 1,
1479 BLK_INTEGRITY_DEVICE_CAPABLE = 1 << 2,
1480 BLK_INTEGRITY_IP_CHECKSUM = 1 << 3,
1481};
1482
1483struct blk_integrity_iter {
1484 void *prot_buf;
1485 void *data_buf;
1486 sector_t seed;
1487 unsigned int data_size;
1488 unsigned short interval;
1489 const char *disk_name;
1490};
1491
1492typedef int (integrity_processing_fn) (struct blk_integrity_iter *);
1493
1494struct blk_integrity_profile {
1495 integrity_processing_fn *generate_fn;
1496 integrity_processing_fn *verify_fn;
1497 const char *name;
1498};
1499
1500extern void blk_integrity_register(struct gendisk *, struct blk_integrity *);
1501extern void blk_integrity_unregister(struct gendisk *);
1502extern int blk_integrity_compare(struct gendisk *, struct gendisk *);
1503extern int blk_rq_map_integrity_sg(struct request_queue *, struct bio *,
1504 struct scatterlist *);
1505extern int blk_rq_count_integrity_sg(struct request_queue *, struct bio *);
1506extern bool blk_integrity_merge_rq(struct request_queue *, struct request *,
1507 struct request *);
1508extern bool blk_integrity_merge_bio(struct request_queue *, struct request *,
1509 struct bio *);
1510
1511static inline struct blk_integrity *blk_get_integrity(struct gendisk *disk)
1512{
1513 struct blk_integrity *bi = &disk->queue->integrity;
1514
1515 if (!bi->profile)
1516 return NULL;
1517
1518 return bi;
1519}
1520
1521static inline
1522struct blk_integrity *bdev_get_integrity(struct block_device *bdev)
1523{
1524 return blk_get_integrity(bdev->bd_disk);
1525}
1526
1527static inline bool blk_integrity_rq(struct request *rq)
1528{
1529 return rq->cmd_flags & REQ_INTEGRITY;
1530}
1531
1532static inline void blk_queue_max_integrity_segments(struct request_queue *q,
1533 unsigned int segs)
1534{
1535 q->limits.max_integrity_segments = segs;
1536}
1537
1538static inline unsigned short
1539queue_max_integrity_segments(struct request_queue *q)
1540{
1541 return q->limits.max_integrity_segments;
1542}
1543
1544static inline bool integrity_req_gap_back_merge(struct request *req,
1545 struct bio *next)
1546{
1547 struct bio_integrity_payload *bip = bio_integrity(req->bio);
1548 struct bio_integrity_payload *bip_next = bio_integrity(next);
1549
1550 return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1],
1551 bip_next->bip_vec[0].bv_offset);
1552}
1553
1554static inline bool integrity_req_gap_front_merge(struct request *req,
1555 struct bio *bio)
1556{
1557 struct bio_integrity_payload *bip = bio_integrity(bio);
1558 struct bio_integrity_payload *bip_next = bio_integrity(req->bio);
1559
1560 return bvec_gap_to_prev(req->q, &bip->bip_vec[bip->bip_vcnt - 1],
1561 bip_next->bip_vec[0].bv_offset);
1562}
1563
1564#else
1565
1566struct bio;
1567struct block_device;
1568struct gendisk;
1569struct blk_integrity;
1570
1571static inline int blk_integrity_rq(struct request *rq)
1572{
1573 return 0;
1574}
1575static inline int blk_rq_count_integrity_sg(struct request_queue *q,
1576 struct bio *b)
1577{
1578 return 0;
1579}
1580static inline int blk_rq_map_integrity_sg(struct request_queue *q,
1581 struct bio *b,
1582 struct scatterlist *s)
1583{
1584 return 0;
1585}
1586static inline struct blk_integrity *bdev_get_integrity(struct block_device *b)
1587{
1588 return NULL;
1589}
1590static inline struct blk_integrity *blk_get_integrity(struct gendisk *disk)
1591{
1592 return NULL;
1593}
1594static inline int blk_integrity_compare(struct gendisk *a, struct gendisk *b)
1595{
1596 return 0;
1597}
1598static inline void blk_integrity_register(struct gendisk *d,
1599 struct blk_integrity *b)
1600{
1601}
1602static inline void blk_integrity_unregister(struct gendisk *d)
1603{
1604}
1605static inline void blk_queue_max_integrity_segments(struct request_queue *q,
1606 unsigned int segs)
1607{
1608}
1609static inline unsigned short queue_max_integrity_segments(struct request_queue *q)
1610{
1611 return 0;
1612}
1613static inline bool blk_integrity_merge_rq(struct request_queue *rq,
1614 struct request *r1,
1615 struct request *r2)
1616{
1617 return true;
1618}
1619static inline bool blk_integrity_merge_bio(struct request_queue *rq,
1620 struct request *r,
1621 struct bio *b)
1622{
1623 return true;
1624}
1625
1626static inline bool integrity_req_gap_back_merge(struct request *req,
1627 struct bio *next)
1628{
1629 return false;
1630}
1631static inline bool integrity_req_gap_front_merge(struct request *req,
1632 struct bio *bio)
1633{
1634 return false;
1635}
1636
1637#endif
1638
1639
1640
1641
1642
1643
1644
1645
1646struct blk_dax_ctl {
1647 sector_t sector;
1648 void __pmem *addr;
1649 long size;
1650 pfn_t pfn;
1651};
1652
1653struct block_device_operations {
1654 int (*open) (struct block_device *, fmode_t);
1655 void (*release) (struct gendisk *, fmode_t);
1656 int (*rw_page)(struct block_device *, sector_t, struct page *, int rw);
1657 int (*ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
1658 int (*compat_ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
1659 long (*direct_access)(struct block_device *, sector_t, void __pmem **,
1660 pfn_t *);
1661 unsigned int (*check_events) (struct gendisk *disk,
1662 unsigned int clearing);
1663
1664 int (*media_changed) (struct gendisk *);
1665 void (*unlock_native_capacity) (struct gendisk *);
1666 int (*revalidate_disk) (struct gendisk *);
1667 int (*getgeo)(struct block_device *, struct hd_geometry *);
1668
1669 void (*swap_slot_free_notify) (struct block_device *, unsigned long);
1670 struct module *owner;
1671 const struct pr_ops *pr_ops;
1672};
1673
1674extern int __blkdev_driver_ioctl(struct block_device *, fmode_t, unsigned int,
1675 unsigned long);
1676extern int bdev_read_page(struct block_device *, sector_t, struct page *);
1677extern int bdev_write_page(struct block_device *, sector_t, struct page *,
1678 struct writeback_control *);
1679extern long bdev_direct_access(struct block_device *, struct blk_dax_ctl *);
1680#else
1681
1682struct block_device;
1683
1684
1685
1686
1687#define buffer_heads_over_limit 0
1688
1689static inline long nr_blockdev_pages(void)
1690{
1691 return 0;
1692}
1693
1694struct blk_plug {
1695};
1696
1697static inline void blk_start_plug(struct blk_plug *plug)
1698{
1699}
1700
1701static inline void blk_finish_plug(struct blk_plug *plug)
1702{
1703}
1704
1705static inline void blk_flush_plug(struct task_struct *task)
1706{
1707}
1708
1709static inline void blk_schedule_flush_plug(struct task_struct *task)
1710{
1711}
1712
1713
1714static inline bool blk_needs_flush_plug(struct task_struct *tsk)
1715{
1716 return false;
1717}
1718
1719static inline int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask,
1720 sector_t *error_sector)
1721{
1722 return 0;
1723}
1724
1725#endif
1726
1727#endif
1728