1
2#ifndef _LINUX_BLKDEV_H
3#define _LINUX_BLKDEV_H
4
5#include <linux/sched.h>
6#include <linux/sched/clock.h>
7#include <linux/major.h>
8#include <linux/genhd.h>
9#include <linux/list.h>
10#include <linux/llist.h>
11#include <linux/minmax.h>
12#include <linux/timer.h>
13#include <linux/workqueue.h>
14#include <linux/wait.h>
15#include <linux/mempool.h>
16#include <linux/pfn.h>
17#include <linux/bio.h>
18#include <linux/stringify.h>
19#include <linux/gfp.h>
20#include <linux/smp.h>
21#include <linux/rcupdate.h>
22#include <linux/percpu-refcount.h>
23#include <linux/scatterlist.h>
24#include <linux/blkzoned.h>
25#include <linux/pm.h>
26#include <linux/sbitmap.h>
27
28struct module;
29struct request_queue;
30struct elevator_queue;
31struct blk_trace;
32struct request;
33struct sg_io_hdr;
34struct blkcg_gq;
35struct blk_flush_queue;
36struct pr_ops;
37struct rq_qos;
38struct blk_queue_stats;
39struct blk_stat_callback;
40struct blk_keyslot_manager;
41
42#define BLKDEV_MIN_RQ 4
43#define BLKDEV_MAX_RQ 128
44
45
46#define BLK_MQ_POLL_STATS_BKTS 16
47
48
49#define BLK_MQ_POLL_CLASSIC -1
50
51
52
53
54
55#define BLKCG_MAX_POLS 6
56
57typedef void (rq_end_io_fn)(struct request *, blk_status_t);
58
59
60
61typedef __u32 __bitwise req_flags_t;
62
63
64#define RQF_STARTED ((__force req_flags_t)(1 << 1))
65
66#define RQF_SOFTBARRIER ((__force req_flags_t)(1 << 3))
67
68#define RQF_FLUSH_SEQ ((__force req_flags_t)(1 << 4))
69
70#define RQF_MIXED_MERGE ((__force req_flags_t)(1 << 5))
71
72#define RQF_MQ_INFLIGHT ((__force req_flags_t)(1 << 6))
73
74#define RQF_DONTPREP ((__force req_flags_t)(1 << 7))
75
76#define RQF_FAILED ((__force req_flags_t)(1 << 10))
77
78#define RQF_QUIET ((__force req_flags_t)(1 << 11))
79
80#define RQF_ELVPRIV ((__force req_flags_t)(1 << 12))
81
82#define RQF_IO_STAT ((__force req_flags_t)(1 << 13))
83
84#define RQF_PM ((__force req_flags_t)(1 << 15))
85
86#define RQF_HASHED ((__force req_flags_t)(1 << 16))
87
88#define RQF_STATS ((__force req_flags_t)(1 << 17))
89
90
91#define RQF_SPECIAL_PAYLOAD ((__force req_flags_t)(1 << 18))
92
93#define RQF_ZONE_WRITE_LOCKED ((__force req_flags_t)(1 << 19))
94
95#define RQF_MQ_POLL_SLEPT ((__force req_flags_t)(1 << 20))
96
97#define RQF_TIMED_OUT ((__force req_flags_t)(1 << 21))
98
99
100#define RQF_NOMERGE_FLAGS \
101 (RQF_STARTED | RQF_SOFTBARRIER | RQF_FLUSH_SEQ | RQF_SPECIAL_PAYLOAD)
102
103
104
105
106enum mq_rq_state {
107 MQ_RQ_IDLE = 0,
108 MQ_RQ_IN_FLIGHT = 1,
109 MQ_RQ_COMPLETE = 2,
110};
111
112
113
114
115
116
117
118struct request {
119 struct request_queue *q;
120 struct blk_mq_ctx *mq_ctx;
121 struct blk_mq_hw_ctx *mq_hctx;
122
123 unsigned int cmd_flags;
124 req_flags_t rq_flags;
125
126 int tag;
127 int internal_tag;
128
129
130 unsigned int __data_len;
131 sector_t __sector;
132
133 struct bio *bio;
134 struct bio *biotail;
135
136 struct list_head queuelist;
137
138
139
140
141
142
143
144
145 union {
146 struct hlist_node hash;
147 struct llist_node ipi_list;
148 };
149
150
151
152
153
154
155 union {
156 struct rb_node rb_node;
157 struct bio_vec special_vec;
158 void *completion_data;
159 int error_count;
160 };
161
162
163
164
165
166
167
168 union {
169 struct {
170 struct io_cq *icq;
171 void *priv[2];
172 } elv;
173
174 struct {
175 unsigned int seq;
176 struct list_head list;
177 rq_end_io_fn *saved_end_io;
178 } flush;
179 };
180
181 struct gendisk *rq_disk;
182 struct block_device *part;
183#ifdef CONFIG_BLK_RQ_ALLOC_TIME
184
185 u64 alloc_time_ns;
186#endif
187
188 u64 start_time_ns;
189
190 u64 io_start_time_ns;
191
192#ifdef CONFIG_BLK_WBT
193 unsigned short wbt_flags;
194#endif
195
196
197
198
199
200 unsigned short stats_sectors;
201
202
203
204
205
206 unsigned short nr_phys_segments;
207
208#if defined(CONFIG_BLK_DEV_INTEGRITY)
209 unsigned short nr_integrity_segments;
210#endif
211
212#ifdef CONFIG_BLK_INLINE_ENCRYPTION
213 struct bio_crypt_ctx *crypt_ctx;
214 struct blk_ksm_keyslot *crypt_keyslot;
215#endif
216
217 unsigned short write_hint;
218 unsigned short ioprio;
219
220 enum mq_rq_state state;
221 refcount_t ref;
222
223 unsigned int timeout;
224 unsigned long deadline;
225
226 union {
227 struct __call_single_data csd;
228 u64 fifo_time;
229 };
230
231
232
233
234 rq_end_io_fn *end_io;
235 void *end_io_data;
236};
237
238static inline bool blk_op_is_passthrough(unsigned int op)
239{
240 op &= REQ_OP_MASK;
241 return op == REQ_OP_DRV_IN || op == REQ_OP_DRV_OUT;
242}
243
244static inline bool blk_rq_is_passthrough(struct request *rq)
245{
246 return blk_op_is_passthrough(req_op(rq));
247}
248
249static inline unsigned short req_get_ioprio(struct request *req)
250{
251 return req->ioprio;
252}
253
254#include <linux/elevator.h>
255
256struct blk_queue_ctx;
257
258struct bio_vec;
259
260enum blk_eh_timer_return {
261 BLK_EH_DONE,
262 BLK_EH_RESET_TIMER,
263};
264
265enum blk_queue_state {
266 Queue_down,
267 Queue_up,
268};
269
270#define BLK_TAG_ALLOC_FIFO 0
271#define BLK_TAG_ALLOC_RR 1
272
273
274
275
276
277
278
279enum blk_zoned_model {
280 BLK_ZONED_NONE = 0,
281 BLK_ZONED_HA,
282 BLK_ZONED_HM,
283};
284
285
286
287
288
289enum blk_bounce {
290 BLK_BOUNCE_NONE,
291 BLK_BOUNCE_HIGH,
292};
293
294struct queue_limits {
295 enum blk_bounce bounce;
296 unsigned long seg_boundary_mask;
297 unsigned long virt_boundary_mask;
298
299 unsigned int max_hw_sectors;
300 unsigned int max_dev_sectors;
301 unsigned int chunk_sectors;
302 unsigned int max_sectors;
303 unsigned int max_segment_size;
304 unsigned int physical_block_size;
305 unsigned int logical_block_size;
306 unsigned int alignment_offset;
307 unsigned int io_min;
308 unsigned int io_opt;
309 unsigned int max_discard_sectors;
310 unsigned int max_hw_discard_sectors;
311 unsigned int max_write_same_sectors;
312 unsigned int max_write_zeroes_sectors;
313 unsigned int max_zone_append_sectors;
314 unsigned int discard_granularity;
315 unsigned int discard_alignment;
316 unsigned int zone_write_granularity;
317
318 unsigned short max_segments;
319 unsigned short max_integrity_segments;
320 unsigned short max_discard_segments;
321
322 unsigned char misaligned;
323 unsigned char discard_misaligned;
324 unsigned char raid_partial_stripes_expensive;
325 enum blk_zoned_model zoned;
326};
327
328typedef int (*report_zones_cb)(struct blk_zone *zone, unsigned int idx,
329 void *data);
330
331void blk_queue_set_zoned(struct gendisk *disk, enum blk_zoned_model model);
332
333#ifdef CONFIG_BLK_DEV_ZONED
334
335#define BLK_ALL_ZONES ((unsigned int)-1)
336int blkdev_report_zones(struct block_device *bdev, sector_t sector,
337 unsigned int nr_zones, report_zones_cb cb, void *data);
338unsigned int blkdev_nr_zones(struct gendisk *disk);
339extern int blkdev_zone_mgmt(struct block_device *bdev, enum req_opf op,
340 sector_t sectors, sector_t nr_sectors,
341 gfp_t gfp_mask);
342int blk_revalidate_disk_zones(struct gendisk *disk,
343 void (*update_driver_data)(struct gendisk *disk));
344
345extern int blkdev_report_zones_ioctl(struct block_device *bdev, fmode_t mode,
346 unsigned int cmd, unsigned long arg);
347extern int blkdev_zone_mgmt_ioctl(struct block_device *bdev, fmode_t mode,
348 unsigned int cmd, unsigned long arg);
349
350#else
351
352static inline unsigned int blkdev_nr_zones(struct gendisk *disk)
353{
354 return 0;
355}
356
357static inline int blkdev_report_zones_ioctl(struct block_device *bdev,
358 fmode_t mode, unsigned int cmd,
359 unsigned long arg)
360{
361 return -ENOTTY;
362}
363
364static inline int blkdev_zone_mgmt_ioctl(struct block_device *bdev,
365 fmode_t mode, unsigned int cmd,
366 unsigned long arg)
367{
368 return -ENOTTY;
369}
370
371#endif
372
373struct request_queue {
374 struct request *last_merge;
375 struct elevator_queue *elevator;
376
377 struct percpu_ref q_usage_counter;
378
379 struct blk_queue_stats *stats;
380 struct rq_qos *rq_qos;
381
382 const struct blk_mq_ops *mq_ops;
383
384
385 struct blk_mq_ctx __percpu *queue_ctx;
386
387 unsigned int queue_depth;
388
389
390 struct blk_mq_hw_ctx **queue_hw_ctx;
391 unsigned int nr_hw_queues;
392
393
394
395
396
397 void *queuedata;
398
399
400
401
402 unsigned long queue_flags;
403
404
405
406
407 atomic_t pm_only;
408
409
410
411
412
413 int id;
414
415 spinlock_t queue_lock;
416
417 struct gendisk *disk;
418
419
420
421
422 struct kobject kobj;
423
424
425
426
427 struct kobject *mq_kobj;
428
429#ifdef CONFIG_BLK_DEV_INTEGRITY
430 struct blk_integrity integrity;
431#endif
432
433#ifdef CONFIG_PM
434 struct device *dev;
435 enum rpm_status rpm_status;
436#endif
437
438
439
440
441 unsigned long nr_requests;
442
443 unsigned int dma_pad_mask;
444 unsigned int dma_alignment;
445
446#ifdef CONFIG_BLK_INLINE_ENCRYPTION
447
448 struct blk_keyslot_manager *ksm;
449#endif
450
451 unsigned int rq_timeout;
452 int poll_nsec;
453
454 struct blk_stat_callback *poll_cb;
455 struct blk_rq_stat poll_stat[BLK_MQ_POLL_STATS_BKTS];
456
457 struct timer_list timeout;
458 struct work_struct timeout_work;
459
460 atomic_t nr_active_requests_shared_sbitmap;
461
462 struct sbitmap_queue sched_bitmap_tags;
463 struct sbitmap_queue sched_breserved_tags;
464
465 struct list_head icq_list;
466#ifdef CONFIG_BLK_CGROUP
467 DECLARE_BITMAP (blkcg_pols, BLKCG_MAX_POLS);
468 struct blkcg_gq *root_blkg;
469 struct list_head blkg_list;
470#endif
471
472 struct queue_limits limits;
473
474 unsigned int required_elevator_features;
475
476#ifdef CONFIG_BLK_DEV_ZONED
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494 unsigned int nr_zones;
495 unsigned long *conv_zones_bitmap;
496 unsigned long *seq_zones_wlock;
497 unsigned int max_open_zones;
498 unsigned int max_active_zones;
499#endif
500
501 int node;
502 struct mutex debugfs_mutex;
503#ifdef CONFIG_BLK_DEV_IO_TRACE
504 struct blk_trace __rcu *blk_trace;
505#endif
506
507
508
509 struct blk_flush_queue *fq;
510
511 struct list_head requeue_list;
512 spinlock_t requeue_lock;
513 struct delayed_work requeue_work;
514
515 struct mutex sysfs_lock;
516 struct mutex sysfs_dir_lock;
517
518
519
520
521
522 struct list_head unused_hctx_list;
523 spinlock_t unused_hctx_lock;
524
525 int mq_freeze_depth;
526
527#ifdef CONFIG_BLK_DEV_THROTTLING
528
529 struct throtl_data *td;
530#endif
531 struct rcu_head rcu_head;
532 wait_queue_head_t mq_freeze_wq;
533
534
535
536
537 struct mutex mq_freeze_lock;
538
539 struct blk_mq_tag_set *tag_set;
540 struct list_head tag_set_list;
541 struct bio_set bio_split;
542
543 struct dentry *debugfs_dir;
544
545#ifdef CONFIG_BLK_DEBUG_FS
546 struct dentry *sched_debugfs_dir;
547 struct dentry *rqos_debugfs_dir;
548#endif
549
550 bool mq_sysfs_init_done;
551
552 size_t cmd_size;
553
554#define BLK_MAX_WRITE_HINTS 5
555 u64 write_hints[BLK_MAX_WRITE_HINTS];
556};
557
558
559#define QUEUE_FLAG_STOPPED 0
560#define QUEUE_FLAG_DYING 1
561#define QUEUE_FLAG_NOMERGES 3
562#define QUEUE_FLAG_SAME_COMP 4
563#define QUEUE_FLAG_FAIL_IO 5
564#define QUEUE_FLAG_NONROT 6
565#define QUEUE_FLAG_VIRT QUEUE_FLAG_NONROT
566#define QUEUE_FLAG_IO_STAT 7
567#define QUEUE_FLAG_DISCARD 8
568#define QUEUE_FLAG_NOXMERGES 9
569#define QUEUE_FLAG_ADD_RANDOM 10
570#define QUEUE_FLAG_SECERASE 11
571#define QUEUE_FLAG_SAME_FORCE 12
572#define QUEUE_FLAG_DEAD 13
573#define QUEUE_FLAG_INIT_DONE 14
574#define QUEUE_FLAG_STABLE_WRITES 15
575#define QUEUE_FLAG_POLL 16
576#define QUEUE_FLAG_WC 17
577#define QUEUE_FLAG_FUA 18
578#define QUEUE_FLAG_DAX 19
579#define QUEUE_FLAG_STATS 20
580#define QUEUE_FLAG_POLL_STATS 21
581#define QUEUE_FLAG_REGISTERED 22
582#define QUEUE_FLAG_SCSI_PASSTHROUGH 23
583#define QUEUE_FLAG_QUIESCED 24
584#define QUEUE_FLAG_PCI_P2PDMA 25
585#define QUEUE_FLAG_ZONE_RESETALL 26
586#define QUEUE_FLAG_RQ_ALLOC_TIME 27
587#define QUEUE_FLAG_HCTX_ACTIVE 28
588#define QUEUE_FLAG_NOWAIT 29
589
590#define QUEUE_FLAG_MQ_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \
591 (1 << QUEUE_FLAG_SAME_COMP) | \
592 (1 << QUEUE_FLAG_NOWAIT))
593
594void blk_queue_flag_set(unsigned int flag, struct request_queue *q);
595void blk_queue_flag_clear(unsigned int flag, struct request_queue *q);
596bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q);
597
598#define blk_queue_stopped(q) test_bit(QUEUE_FLAG_STOPPED, &(q)->queue_flags)
599#define blk_queue_dying(q) test_bit(QUEUE_FLAG_DYING, &(q)->queue_flags)
600#define blk_queue_dead(q) test_bit(QUEUE_FLAG_DEAD, &(q)->queue_flags)
601#define blk_queue_init_done(q) test_bit(QUEUE_FLAG_INIT_DONE, &(q)->queue_flags)
602#define blk_queue_nomerges(q) test_bit(QUEUE_FLAG_NOMERGES, &(q)->queue_flags)
603#define blk_queue_noxmerges(q) \
604 test_bit(QUEUE_FLAG_NOXMERGES, &(q)->queue_flags)
605#define blk_queue_nonrot(q) test_bit(QUEUE_FLAG_NONROT, &(q)->queue_flags)
606#define blk_queue_stable_writes(q) \
607 test_bit(QUEUE_FLAG_STABLE_WRITES, &(q)->queue_flags)
608#define blk_queue_io_stat(q) test_bit(QUEUE_FLAG_IO_STAT, &(q)->queue_flags)
609#define blk_queue_add_random(q) test_bit(QUEUE_FLAG_ADD_RANDOM, &(q)->queue_flags)
610#define blk_queue_discard(q) test_bit(QUEUE_FLAG_DISCARD, &(q)->queue_flags)
611#define blk_queue_zone_resetall(q) \
612 test_bit(QUEUE_FLAG_ZONE_RESETALL, &(q)->queue_flags)
613#define blk_queue_secure_erase(q) \
614 (test_bit(QUEUE_FLAG_SECERASE, &(q)->queue_flags))
615#define blk_queue_dax(q) test_bit(QUEUE_FLAG_DAX, &(q)->queue_flags)
616#define blk_queue_scsi_passthrough(q) \
617 test_bit(QUEUE_FLAG_SCSI_PASSTHROUGH, &(q)->queue_flags)
618#define blk_queue_pci_p2pdma(q) \
619 test_bit(QUEUE_FLAG_PCI_P2PDMA, &(q)->queue_flags)
620#ifdef CONFIG_BLK_RQ_ALLOC_TIME
621#define blk_queue_rq_alloc_time(q) \
622 test_bit(QUEUE_FLAG_RQ_ALLOC_TIME, &(q)->queue_flags)
623#else
624#define blk_queue_rq_alloc_time(q) false
625#endif
626
627#define blk_noretry_request(rq) \
628 ((rq)->cmd_flags & (REQ_FAILFAST_DEV|REQ_FAILFAST_TRANSPORT| \
629 REQ_FAILFAST_DRIVER))
630#define blk_queue_quiesced(q) test_bit(QUEUE_FLAG_QUIESCED, &(q)->queue_flags)
631#define blk_queue_pm_only(q) atomic_read(&(q)->pm_only)
632#define blk_queue_fua(q) test_bit(QUEUE_FLAG_FUA, &(q)->queue_flags)
633#define blk_queue_registered(q) test_bit(QUEUE_FLAG_REGISTERED, &(q)->queue_flags)
634#define blk_queue_nowait(q) test_bit(QUEUE_FLAG_NOWAIT, &(q)->queue_flags)
635
636extern void blk_set_pm_only(struct request_queue *q);
637extern void blk_clear_pm_only(struct request_queue *q);
638
639#define list_entry_rq(ptr) list_entry((ptr), struct request, queuelist)
640
641#define rq_data_dir(rq) (op_is_write(req_op(rq)) ? WRITE : READ)
642
643#define rq_dma_dir(rq) \
644 (op_is_write(req_op(rq)) ? DMA_TO_DEVICE : DMA_FROM_DEVICE)
645
646#define dma_map_bvec(dev, bv, dir, attrs) \
647 dma_map_page_attrs(dev, (bv)->bv_page, (bv)->bv_offset, (bv)->bv_len, \
648 (dir), (attrs))
649
650static inline bool queue_is_mq(struct request_queue *q)
651{
652 return q->mq_ops;
653}
654
655#ifdef CONFIG_PM
656static inline enum rpm_status queue_rpm_status(struct request_queue *q)
657{
658 return q->rpm_status;
659}
660#else
661static inline enum rpm_status queue_rpm_status(struct request_queue *q)
662{
663 return RPM_ACTIVE;
664}
665#endif
666
667static inline enum blk_zoned_model
668blk_queue_zoned_model(struct request_queue *q)
669{
670 if (IS_ENABLED(CONFIG_BLK_DEV_ZONED))
671 return q->limits.zoned;
672 return BLK_ZONED_NONE;
673}
674
675static inline bool blk_queue_is_zoned(struct request_queue *q)
676{
677 switch (blk_queue_zoned_model(q)) {
678 case BLK_ZONED_HA:
679 case BLK_ZONED_HM:
680 return true;
681 default:
682 return false;
683 }
684}
685
686static inline sector_t blk_queue_zone_sectors(struct request_queue *q)
687{
688 return blk_queue_is_zoned(q) ? q->limits.chunk_sectors : 0;
689}
690
691#ifdef CONFIG_BLK_DEV_ZONED
692static inline unsigned int blk_queue_nr_zones(struct request_queue *q)
693{
694 return blk_queue_is_zoned(q) ? q->nr_zones : 0;
695}
696
697static inline unsigned int blk_queue_zone_no(struct request_queue *q,
698 sector_t sector)
699{
700 if (!blk_queue_is_zoned(q))
701 return 0;
702 return sector >> ilog2(q->limits.chunk_sectors);
703}
704
705static inline bool blk_queue_zone_is_seq(struct request_queue *q,
706 sector_t sector)
707{
708 if (!blk_queue_is_zoned(q))
709 return false;
710 if (!q->conv_zones_bitmap)
711 return true;
712 return !test_bit(blk_queue_zone_no(q, sector), q->conv_zones_bitmap);
713}
714
715static inline void blk_queue_max_open_zones(struct request_queue *q,
716 unsigned int max_open_zones)
717{
718 q->max_open_zones = max_open_zones;
719}
720
721static inline unsigned int queue_max_open_zones(const struct request_queue *q)
722{
723 return q->max_open_zones;
724}
725
726static inline void blk_queue_max_active_zones(struct request_queue *q,
727 unsigned int max_active_zones)
728{
729 q->max_active_zones = max_active_zones;
730}
731
732static inline unsigned int queue_max_active_zones(const struct request_queue *q)
733{
734 return q->max_active_zones;
735}
736#else
737static inline unsigned int blk_queue_nr_zones(struct request_queue *q)
738{
739 return 0;
740}
741static inline bool blk_queue_zone_is_seq(struct request_queue *q,
742 sector_t sector)
743{
744 return false;
745}
746static inline unsigned int blk_queue_zone_no(struct request_queue *q,
747 sector_t sector)
748{
749 return 0;
750}
751static inline unsigned int queue_max_open_zones(const struct request_queue *q)
752{
753 return 0;
754}
755static inline unsigned int queue_max_active_zones(const struct request_queue *q)
756{
757 return 0;
758}
759#endif
760
761static inline bool rq_is_sync(struct request *rq)
762{
763 return op_is_sync(rq->cmd_flags);
764}
765
766static inline bool rq_mergeable(struct request *rq)
767{
768 if (blk_rq_is_passthrough(rq))
769 return false;
770
771 if (req_op(rq) == REQ_OP_FLUSH)
772 return false;
773
774 if (req_op(rq) == REQ_OP_WRITE_ZEROES)
775 return false;
776
777 if (req_op(rq) == REQ_OP_ZONE_APPEND)
778 return false;
779
780 if (rq->cmd_flags & REQ_NOMERGE_FLAGS)
781 return false;
782 if (rq->rq_flags & RQF_NOMERGE_FLAGS)
783 return false;
784
785 return true;
786}
787
788static inline bool blk_write_same_mergeable(struct bio *a, struct bio *b)
789{
790 if (bio_page(a) == bio_page(b) &&
791 bio_offset(a) == bio_offset(b))
792 return true;
793
794 return false;
795}
796
797static inline unsigned int blk_queue_depth(struct request_queue *q)
798{
799 if (q->queue_depth)
800 return q->queue_depth;
801
802 return q->nr_requests;
803}
804
805
806
807
808#define BLK_DEFAULT_SG_TIMEOUT (60 * HZ)
809#define BLK_MIN_SG_TIMEOUT (7 * HZ)
810
811struct rq_map_data {
812 struct page **pages;
813 int page_order;
814 int nr_entries;
815 unsigned long offset;
816 int null_mapped;
817 int from_user;
818};
819
820struct req_iterator {
821 struct bvec_iter iter;
822 struct bio *bio;
823};
824
825
826#define for_each_bio(_bio) \
827 for (; _bio; _bio = _bio->bi_next)
828#define __rq_for_each_bio(_bio, rq) \
829 if ((rq->bio)) \
830 for (_bio = (rq)->bio; _bio; _bio = _bio->bi_next)
831
832#define rq_for_each_segment(bvl, _rq, _iter) \
833 __rq_for_each_bio(_iter.bio, _rq) \
834 bio_for_each_segment(bvl, _iter.bio, _iter.iter)
835
836#define rq_for_each_bvec(bvl, _rq, _iter) \
837 __rq_for_each_bio(_iter.bio, _rq) \
838 bio_for_each_bvec(bvl, _iter.bio, _iter.iter)
839
840#define rq_iter_last(bvec, _iter) \
841 (_iter.bio->bi_next == NULL && \
842 bio_iter_last(bvec, _iter.iter))
843
844#ifndef ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
845# error "You should define ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE for your platform"
846#endif
847#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
848extern void rq_flush_dcache_pages(struct request *rq);
849#else
850static inline void rq_flush_dcache_pages(struct request *rq)
851{
852}
853#endif
854
855extern int blk_register_queue(struct gendisk *disk);
856extern void blk_unregister_queue(struct gendisk *disk);
857blk_qc_t submit_bio_noacct(struct bio *bio);
858extern void blk_rq_init(struct request_queue *q, struct request *rq);
859extern void blk_put_request(struct request *);
860extern struct request *blk_get_request(struct request_queue *, unsigned int op,
861 blk_mq_req_flags_t flags);
862extern int blk_lld_busy(struct request_queue *q);
863extern int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
864 struct bio_set *bs, gfp_t gfp_mask,
865 int (*bio_ctr)(struct bio *, struct bio *, void *),
866 void *data);
867extern void blk_rq_unprep_clone(struct request *rq);
868extern blk_status_t blk_insert_cloned_request(struct request_queue *q,
869 struct request *rq);
870int blk_rq_append_bio(struct request *rq, struct bio *bio);
871extern void blk_queue_split(struct bio **);
872extern int blk_queue_enter(struct request_queue *q, blk_mq_req_flags_t flags);
873extern void blk_queue_exit(struct request_queue *q);
874extern void blk_sync_queue(struct request_queue *q);
875extern int blk_rq_map_user(struct request_queue *, struct request *,
876 struct rq_map_data *, void __user *, unsigned long,
877 gfp_t);
878extern int blk_rq_unmap_user(struct bio *);
879extern int blk_rq_map_kern(struct request_queue *, struct request *, void *, unsigned int, gfp_t);
880extern int blk_rq_map_user_iov(struct request_queue *, struct request *,
881 struct rq_map_data *, const struct iov_iter *,
882 gfp_t);
883extern void blk_execute_rq_nowait(struct gendisk *,
884 struct request *, int, rq_end_io_fn *);
885
886blk_status_t blk_execute_rq(struct gendisk *bd_disk, struct request *rq,
887 int at_head);
888
889
890extern const char *blk_op_str(unsigned int op);
891
892int blk_status_to_errno(blk_status_t status);
893blk_status_t errno_to_blk_status(int errno);
894
895int blk_poll(struct request_queue *q, blk_qc_t cookie, bool spin);
896
897static inline struct request_queue *bdev_get_queue(struct block_device *bdev)
898{
899 return bdev->bd_disk->queue;
900}
901
902
903
904
905
906
907
908#ifndef SECTOR_SHIFT
909#define SECTOR_SHIFT 9
910#endif
911#ifndef SECTOR_SIZE
912#define SECTOR_SIZE (1 << SECTOR_SHIFT)
913#endif
914
915#define PAGE_SECTORS_SHIFT (PAGE_SHIFT - SECTOR_SHIFT)
916#define PAGE_SECTORS (1 << PAGE_SECTORS_SHIFT)
917#define SECTOR_MASK (PAGE_SECTORS - 1)
918
919
920
921
922
923
924
925
926
927
928static inline sector_t blk_rq_pos(const struct request *rq)
929{
930 return rq->__sector;
931}
932
933static inline unsigned int blk_rq_bytes(const struct request *rq)
934{
935 return rq->__data_len;
936}
937
938static inline int blk_rq_cur_bytes(const struct request *rq)
939{
940 return rq->bio ? bio_cur_bytes(rq->bio) : 0;
941}
942
943extern unsigned int blk_rq_err_bytes(const struct request *rq);
944
945static inline unsigned int blk_rq_sectors(const struct request *rq)
946{
947 return blk_rq_bytes(rq) >> SECTOR_SHIFT;
948}
949
950static inline unsigned int blk_rq_cur_sectors(const struct request *rq)
951{
952 return blk_rq_cur_bytes(rq) >> SECTOR_SHIFT;
953}
954
955static inline unsigned int blk_rq_stats_sectors(const struct request *rq)
956{
957 return rq->stats_sectors;
958}
959
960#ifdef CONFIG_BLK_DEV_ZONED
961
962
963const char *blk_zone_cond_str(enum blk_zone_cond zone_cond);
964
965static inline unsigned int bio_zone_no(struct bio *bio)
966{
967 return blk_queue_zone_no(bdev_get_queue(bio->bi_bdev),
968 bio->bi_iter.bi_sector);
969}
970
971static inline unsigned int bio_zone_is_seq(struct bio *bio)
972{
973 return blk_queue_zone_is_seq(bdev_get_queue(bio->bi_bdev),
974 bio->bi_iter.bi_sector);
975}
976
977static inline unsigned int blk_rq_zone_no(struct request *rq)
978{
979 return blk_queue_zone_no(rq->q, blk_rq_pos(rq));
980}
981
982static inline unsigned int blk_rq_zone_is_seq(struct request *rq)
983{
984 return blk_queue_zone_is_seq(rq->q, blk_rq_pos(rq));
985}
986#endif
987
988
989
990
991
992
993
994static inline unsigned int blk_rq_payload_bytes(struct request *rq)
995{
996 if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)
997 return rq->special_vec.bv_len;
998 return blk_rq_bytes(rq);
999}
1000
1001
1002
1003
1004
1005static inline struct bio_vec req_bvec(struct request *rq)
1006{
1007 if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)
1008 return rq->special_vec;
1009 return mp_bvec_iter_bvec(rq->bio->bi_io_vec, rq->bio->bi_iter);
1010}
1011
1012static inline unsigned int blk_queue_get_max_sectors(struct request_queue *q,
1013 int op)
1014{
1015 if (unlikely(op == REQ_OP_DISCARD || op == REQ_OP_SECURE_ERASE))
1016 return min(q->limits.max_discard_sectors,
1017 UINT_MAX >> SECTOR_SHIFT);
1018
1019 if (unlikely(op == REQ_OP_WRITE_SAME))
1020 return q->limits.max_write_same_sectors;
1021
1022 if (unlikely(op == REQ_OP_WRITE_ZEROES))
1023 return q->limits.max_write_zeroes_sectors;
1024
1025 return q->limits.max_sectors;
1026}
1027
1028
1029
1030
1031
1032static inline unsigned int blk_max_size_offset(struct request_queue *q,
1033 sector_t offset,
1034 unsigned int chunk_sectors)
1035{
1036 if (!chunk_sectors) {
1037 if (q->limits.chunk_sectors)
1038 chunk_sectors = q->limits.chunk_sectors;
1039 else
1040 return q->limits.max_sectors;
1041 }
1042
1043 if (likely(is_power_of_2(chunk_sectors)))
1044 chunk_sectors -= offset & (chunk_sectors - 1);
1045 else
1046 chunk_sectors -= sector_div(offset, chunk_sectors);
1047
1048 return min(q->limits.max_sectors, chunk_sectors);
1049}
1050
1051static inline unsigned int blk_rq_get_max_sectors(struct request *rq,
1052 sector_t offset)
1053{
1054 struct request_queue *q = rq->q;
1055
1056 if (blk_rq_is_passthrough(rq))
1057 return q->limits.max_hw_sectors;
1058
1059 if (!q->limits.chunk_sectors ||
1060 req_op(rq) == REQ_OP_DISCARD ||
1061 req_op(rq) == REQ_OP_SECURE_ERASE)
1062 return blk_queue_get_max_sectors(q, req_op(rq));
1063
1064 return min(blk_max_size_offset(q, offset, 0),
1065 blk_queue_get_max_sectors(q, req_op(rq)));
1066}
1067
1068static inline unsigned int blk_rq_count_bios(struct request *rq)
1069{
1070 unsigned int nr_bios = 0;
1071 struct bio *bio;
1072
1073 __rq_for_each_bio(bio, rq)
1074 nr_bios++;
1075
1076 return nr_bios;
1077}
1078
1079void blk_steal_bios(struct bio_list *list, struct request *rq);
1080
1081
1082
1083
1084
1085
1086
1087extern bool blk_update_request(struct request *rq, blk_status_t error,
1088 unsigned int nr_bytes);
1089
1090extern void blk_abort_request(struct request *);
1091
1092
1093
1094
1095extern void blk_cleanup_queue(struct request_queue *);
1096void blk_queue_bounce_limit(struct request_queue *q, enum blk_bounce limit);
1097extern void blk_queue_max_hw_sectors(struct request_queue *, unsigned int);
1098extern void blk_queue_chunk_sectors(struct request_queue *, unsigned int);
1099extern void blk_queue_max_segments(struct request_queue *, unsigned short);
1100extern void blk_queue_max_discard_segments(struct request_queue *,
1101 unsigned short);
1102extern void blk_queue_max_segment_size(struct request_queue *, unsigned int);
1103extern void blk_queue_max_discard_sectors(struct request_queue *q,
1104 unsigned int max_discard_sectors);
1105extern void blk_queue_max_write_same_sectors(struct request_queue *q,
1106 unsigned int max_write_same_sectors);
1107extern void blk_queue_max_write_zeroes_sectors(struct request_queue *q,
1108 unsigned int max_write_same_sectors);
1109extern void blk_queue_logical_block_size(struct request_queue *, unsigned int);
1110extern void blk_queue_max_zone_append_sectors(struct request_queue *q,
1111 unsigned int max_zone_append_sectors);
1112extern void blk_queue_physical_block_size(struct request_queue *, unsigned int);
1113void blk_queue_zone_write_granularity(struct request_queue *q,
1114 unsigned int size);
1115extern void blk_queue_alignment_offset(struct request_queue *q,
1116 unsigned int alignment);
1117void disk_update_readahead(struct gendisk *disk);
1118extern void blk_limits_io_min(struct queue_limits *limits, unsigned int min);
1119extern void blk_queue_io_min(struct request_queue *q, unsigned int min);
1120extern void blk_limits_io_opt(struct queue_limits *limits, unsigned int opt);
1121extern void blk_queue_io_opt(struct request_queue *q, unsigned int opt);
1122extern void blk_set_queue_depth(struct request_queue *q, unsigned int depth);
1123extern void blk_set_default_limits(struct queue_limits *lim);
1124extern void blk_set_stacking_limits(struct queue_limits *lim);
1125extern int blk_stack_limits(struct queue_limits *t, struct queue_limits *b,
1126 sector_t offset);
1127extern void disk_stack_limits(struct gendisk *disk, struct block_device *bdev,
1128 sector_t offset);
1129extern void blk_queue_update_dma_pad(struct request_queue *, unsigned int);
1130extern void blk_queue_segment_boundary(struct request_queue *, unsigned long);
1131extern void blk_queue_virt_boundary(struct request_queue *, unsigned long);
1132extern void blk_queue_dma_alignment(struct request_queue *, int);
1133extern void blk_queue_update_dma_alignment(struct request_queue *, int);
1134extern void blk_queue_rq_timeout(struct request_queue *, unsigned int);
1135extern void blk_queue_write_cache(struct request_queue *q, bool enabled, bool fua);
1136extern void blk_queue_required_elevator_features(struct request_queue *q,
1137 unsigned int features);
1138extern bool blk_queue_can_use_dma_map_merging(struct request_queue *q,
1139 struct device *dev);
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150static inline unsigned short blk_rq_nr_phys_segments(struct request *rq)
1151{
1152 if (rq->rq_flags & RQF_SPECIAL_PAYLOAD)
1153 return 1;
1154 return rq->nr_phys_segments;
1155}
1156
1157
1158
1159
1160
1161static inline unsigned short blk_rq_nr_discard_segments(struct request *rq)
1162{
1163 return max_t(unsigned short, rq->nr_phys_segments, 1);
1164}
1165
1166int __blk_rq_map_sg(struct request_queue *q, struct request *rq,
1167 struct scatterlist *sglist, struct scatterlist **last_sg);
1168static inline int blk_rq_map_sg(struct request_queue *q, struct request *rq,
1169 struct scatterlist *sglist)
1170{
1171 struct scatterlist *last_sg = NULL;
1172
1173 return __blk_rq_map_sg(q, rq, sglist, &last_sg);
1174}
1175extern void blk_dump_rq_flags(struct request *, char *);
1176
1177bool __must_check blk_get_queue(struct request_queue *);
1178extern void blk_put_queue(struct request_queue *);
1179extern void blk_set_queue_dying(struct request_queue *);
1180
1181#ifdef CONFIG_BLOCK
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194struct blk_plug {
1195 struct list_head mq_list;
1196 struct list_head cb_list;
1197 unsigned short rq_count;
1198 bool multiple_queues;
1199 bool nowait;
1200};
1201#define BLK_MAX_REQUEST_COUNT 16
1202#define BLK_PLUG_FLUSH_SIZE (128 * 1024)
1203
1204struct blk_plug_cb;
1205typedef void (*blk_plug_cb_fn)(struct blk_plug_cb *, bool);
1206struct blk_plug_cb {
1207 struct list_head list;
1208 blk_plug_cb_fn callback;
1209 void *data;
1210};
1211extern struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug,
1212 void *data, int size);
1213extern void blk_start_plug(struct blk_plug *);
1214extern void blk_finish_plug(struct blk_plug *);
1215extern void blk_flush_plug_list(struct blk_plug *, bool);
1216
1217static inline void blk_flush_plug(struct task_struct *tsk)
1218{
1219 struct blk_plug *plug = tsk->plug;
1220
1221 if (plug)
1222 blk_flush_plug_list(plug, false);
1223}
1224
1225static inline void blk_schedule_flush_plug(struct task_struct *tsk)
1226{
1227 struct blk_plug *plug = tsk->plug;
1228
1229 if (plug)
1230 blk_flush_plug_list(plug, true);
1231}
1232
1233static inline bool blk_needs_flush_plug(struct task_struct *tsk)
1234{
1235 struct blk_plug *plug = tsk->plug;
1236
1237 return plug &&
1238 (!list_empty(&plug->mq_list) ||
1239 !list_empty(&plug->cb_list));
1240}
1241
1242int blkdev_issue_flush(struct block_device *bdev);
1243long nr_blockdev_pages(void);
1244#else
1245struct blk_plug {
1246};
1247
1248static inline void blk_start_plug(struct blk_plug *plug)
1249{
1250}
1251
1252static inline void blk_finish_plug(struct blk_plug *plug)
1253{
1254}
1255
1256static inline void blk_flush_plug(struct task_struct *task)
1257{
1258}
1259
1260static inline void blk_schedule_flush_plug(struct task_struct *task)
1261{
1262}
1263
1264
1265static inline bool blk_needs_flush_plug(struct task_struct *tsk)
1266{
1267 return false;
1268}
1269
1270static inline int blkdev_issue_flush(struct block_device *bdev)
1271{
1272 return 0;
1273}
1274
1275static inline long nr_blockdev_pages(void)
1276{
1277 return 0;
1278}
1279#endif
1280
1281extern void blk_io_schedule(void);
1282
1283extern int blkdev_issue_write_same(struct block_device *bdev, sector_t sector,
1284 sector_t nr_sects, gfp_t gfp_mask, struct page *page);
1285
1286#define BLKDEV_DISCARD_SECURE (1 << 0)
1287
1288extern int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
1289 sector_t nr_sects, gfp_t gfp_mask, unsigned long flags);
1290extern int __blkdev_issue_discard(struct block_device *bdev, sector_t sector,
1291 sector_t nr_sects, gfp_t gfp_mask, int flags,
1292 struct bio **biop);
1293
1294#define BLKDEV_ZERO_NOUNMAP (1 << 0)
1295#define BLKDEV_ZERO_NOFALLBACK (1 << 1)
1296
1297extern int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
1298 sector_t nr_sects, gfp_t gfp_mask, struct bio **biop,
1299 unsigned flags);
1300extern int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
1301 sector_t nr_sects, gfp_t gfp_mask, unsigned flags);
1302
1303static inline int sb_issue_discard(struct super_block *sb, sector_t block,
1304 sector_t nr_blocks, gfp_t gfp_mask, unsigned long flags)
1305{
1306 return blkdev_issue_discard(sb->s_bdev,
1307 block << (sb->s_blocksize_bits -
1308 SECTOR_SHIFT),
1309 nr_blocks << (sb->s_blocksize_bits -
1310 SECTOR_SHIFT),
1311 gfp_mask, flags);
1312}
1313static inline int sb_issue_zeroout(struct super_block *sb, sector_t block,
1314 sector_t nr_blocks, gfp_t gfp_mask)
1315{
1316 return blkdev_issue_zeroout(sb->s_bdev,
1317 block << (sb->s_blocksize_bits -
1318 SECTOR_SHIFT),
1319 nr_blocks << (sb->s_blocksize_bits -
1320 SECTOR_SHIFT),
1321 gfp_mask, 0);
1322}
1323
1324static inline bool bdev_is_partition(struct block_device *bdev)
1325{
1326 return bdev->bd_partno;
1327}
1328
1329enum blk_default_limits {
1330 BLK_MAX_SEGMENTS = 128,
1331 BLK_SAFE_MAX_SECTORS = 255,
1332 BLK_DEF_MAX_SECTORS = 2560,
1333 BLK_MAX_SEGMENT_SIZE = 65536,
1334 BLK_SEG_BOUNDARY_MASK = 0xFFFFFFFFUL,
1335};
1336
1337static inline unsigned long queue_segment_boundary(const struct request_queue *q)
1338{
1339 return q->limits.seg_boundary_mask;
1340}
1341
1342static inline unsigned long queue_virt_boundary(const struct request_queue *q)
1343{
1344 return q->limits.virt_boundary_mask;
1345}
1346
1347static inline unsigned int queue_max_sectors(const struct request_queue *q)
1348{
1349 return q->limits.max_sectors;
1350}
1351
1352static inline unsigned int queue_max_bytes(struct request_queue *q)
1353{
1354 return min_t(unsigned int, queue_max_sectors(q), INT_MAX >> 9) << 9;
1355}
1356
1357static inline unsigned int queue_max_hw_sectors(const struct request_queue *q)
1358{
1359 return q->limits.max_hw_sectors;
1360}
1361
1362static inline unsigned short queue_max_segments(const struct request_queue *q)
1363{
1364 return q->limits.max_segments;
1365}
1366
1367static inline unsigned short queue_max_discard_segments(const struct request_queue *q)
1368{
1369 return q->limits.max_discard_segments;
1370}
1371
1372static inline unsigned int queue_max_segment_size(const struct request_queue *q)
1373{
1374 return q->limits.max_segment_size;
1375}
1376
1377static inline unsigned int queue_max_zone_append_sectors(const struct request_queue *q)
1378{
1379
1380 const struct queue_limits *l = &q->limits;
1381
1382 return min(l->max_zone_append_sectors, l->max_sectors);
1383}
1384
1385static inline unsigned queue_logical_block_size(const struct request_queue *q)
1386{
1387 int retval = 512;
1388
1389 if (q && q->limits.logical_block_size)
1390 retval = q->limits.logical_block_size;
1391
1392 return retval;
1393}
1394
1395static inline unsigned int bdev_logical_block_size(struct block_device *bdev)
1396{
1397 return queue_logical_block_size(bdev_get_queue(bdev));
1398}
1399
1400static inline unsigned int queue_physical_block_size(const struct request_queue *q)
1401{
1402 return q->limits.physical_block_size;
1403}
1404
1405static inline unsigned int bdev_physical_block_size(struct block_device *bdev)
1406{
1407 return queue_physical_block_size(bdev_get_queue(bdev));
1408}
1409
1410static inline unsigned int queue_io_min(const struct request_queue *q)
1411{
1412 return q->limits.io_min;
1413}
1414
1415static inline int bdev_io_min(struct block_device *bdev)
1416{
1417 return queue_io_min(bdev_get_queue(bdev));
1418}
1419
1420static inline unsigned int queue_io_opt(const struct request_queue *q)
1421{
1422 return q->limits.io_opt;
1423}
1424
1425static inline int bdev_io_opt(struct block_device *bdev)
1426{
1427 return queue_io_opt(bdev_get_queue(bdev));
1428}
1429
1430static inline unsigned int
1431queue_zone_write_granularity(const struct request_queue *q)
1432{
1433 return q->limits.zone_write_granularity;
1434}
1435
1436static inline unsigned int
1437bdev_zone_write_granularity(struct block_device *bdev)
1438{
1439 return queue_zone_write_granularity(bdev_get_queue(bdev));
1440}
1441
1442static inline int queue_alignment_offset(const struct request_queue *q)
1443{
1444 if (q->limits.misaligned)
1445 return -1;
1446
1447 return q->limits.alignment_offset;
1448}
1449
1450static inline int queue_limit_alignment_offset(struct queue_limits *lim, sector_t sector)
1451{
1452 unsigned int granularity = max(lim->physical_block_size, lim->io_min);
1453 unsigned int alignment = sector_div(sector, granularity >> SECTOR_SHIFT)
1454 << SECTOR_SHIFT;
1455
1456 return (granularity + lim->alignment_offset - alignment) % granularity;
1457}
1458
1459static inline int bdev_alignment_offset(struct block_device *bdev)
1460{
1461 struct request_queue *q = bdev_get_queue(bdev);
1462
1463 if (q->limits.misaligned)
1464 return -1;
1465 if (bdev_is_partition(bdev))
1466 return queue_limit_alignment_offset(&q->limits,
1467 bdev->bd_start_sect);
1468 return q->limits.alignment_offset;
1469}
1470
1471static inline int queue_discard_alignment(const struct request_queue *q)
1472{
1473 if (q->limits.discard_misaligned)
1474 return -1;
1475
1476 return q->limits.discard_alignment;
1477}
1478
1479static inline int queue_limit_discard_alignment(struct queue_limits *lim, sector_t sector)
1480{
1481 unsigned int alignment, granularity, offset;
1482
1483 if (!lim->max_discard_sectors)
1484 return 0;
1485
1486
1487 alignment = lim->discard_alignment >> SECTOR_SHIFT;
1488 granularity = lim->discard_granularity >> SECTOR_SHIFT;
1489 if (!granularity)
1490 return 0;
1491
1492
1493 offset = sector_div(sector, granularity);
1494
1495
1496 offset = (granularity + alignment - offset) % granularity;
1497
1498
1499 return offset << SECTOR_SHIFT;
1500}
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510static inline bool blk_discard_mergable(struct request *req)
1511{
1512 if (req_op(req) == REQ_OP_DISCARD &&
1513 queue_max_discard_segments(req->q) > 1)
1514 return true;
1515 return false;
1516}
1517
1518static inline int bdev_discard_alignment(struct block_device *bdev)
1519{
1520 struct request_queue *q = bdev_get_queue(bdev);
1521
1522 if (bdev_is_partition(bdev))
1523 return queue_limit_discard_alignment(&q->limits,
1524 bdev->bd_start_sect);
1525 return q->limits.discard_alignment;
1526}
1527
1528static inline unsigned int bdev_write_same(struct block_device *bdev)
1529{
1530 struct request_queue *q = bdev_get_queue(bdev);
1531
1532 if (q)
1533 return q->limits.max_write_same_sectors;
1534
1535 return 0;
1536}
1537
1538static inline unsigned int bdev_write_zeroes_sectors(struct block_device *bdev)
1539{
1540 struct request_queue *q = bdev_get_queue(bdev);
1541
1542 if (q)
1543 return q->limits.max_write_zeroes_sectors;
1544
1545 return 0;
1546}
1547
1548static inline enum blk_zoned_model bdev_zoned_model(struct block_device *bdev)
1549{
1550 struct request_queue *q = bdev_get_queue(bdev);
1551
1552 if (q)
1553 return blk_queue_zoned_model(q);
1554
1555 return BLK_ZONED_NONE;
1556}
1557
1558static inline bool bdev_is_zoned(struct block_device *bdev)
1559{
1560 struct request_queue *q = bdev_get_queue(bdev);
1561
1562 if (q)
1563 return blk_queue_is_zoned(q);
1564
1565 return false;
1566}
1567
1568static inline sector_t bdev_zone_sectors(struct block_device *bdev)
1569{
1570 struct request_queue *q = bdev_get_queue(bdev);
1571
1572 if (q)
1573 return blk_queue_zone_sectors(q);
1574 return 0;
1575}
1576
1577static inline unsigned int bdev_max_open_zones(struct block_device *bdev)
1578{
1579 struct request_queue *q = bdev_get_queue(bdev);
1580
1581 if (q)
1582 return queue_max_open_zones(q);
1583 return 0;
1584}
1585
1586static inline unsigned int bdev_max_active_zones(struct block_device *bdev)
1587{
1588 struct request_queue *q = bdev_get_queue(bdev);
1589
1590 if (q)
1591 return queue_max_active_zones(q);
1592 return 0;
1593}
1594
1595static inline int queue_dma_alignment(const struct request_queue *q)
1596{
1597 return q ? q->dma_alignment : 511;
1598}
1599
1600static inline int blk_rq_aligned(struct request_queue *q, unsigned long addr,
1601 unsigned int len)
1602{
1603 unsigned int alignment = queue_dma_alignment(q) | q->dma_pad_mask;
1604 return !(addr & alignment) && !(len & alignment);
1605}
1606
1607
1608static inline unsigned int blksize_bits(unsigned int size)
1609{
1610 unsigned int bits = 8;
1611 do {
1612 bits++;
1613 size >>= 1;
1614 } while (size > 256);
1615 return bits;
1616}
1617
1618static inline unsigned int block_size(struct block_device *bdev)
1619{
1620 return 1 << bdev->bd_inode->i_blkbits;
1621}
1622
1623int kblockd_schedule_work(struct work_struct *work);
1624int kblockd_mod_delayed_work_on(int cpu, struct delayed_work *dwork, unsigned long delay);
1625
1626#define MODULE_ALIAS_BLOCKDEV(major,minor) \
1627 MODULE_ALIAS("block-major-" __stringify(major) "-" __stringify(minor))
1628#define MODULE_ALIAS_BLOCKDEV_MAJOR(major) \
1629 MODULE_ALIAS("block-major-" __stringify(major) "-*")
1630
1631#if defined(CONFIG_BLK_DEV_INTEGRITY)
1632
1633enum blk_integrity_flags {
1634 BLK_INTEGRITY_VERIFY = 1 << 0,
1635 BLK_INTEGRITY_GENERATE = 1 << 1,
1636 BLK_INTEGRITY_DEVICE_CAPABLE = 1 << 2,
1637 BLK_INTEGRITY_IP_CHECKSUM = 1 << 3,
1638};
1639
1640struct blk_integrity_iter {
1641 void *prot_buf;
1642 void *data_buf;
1643 sector_t seed;
1644 unsigned int data_size;
1645 unsigned short interval;
1646 const char *disk_name;
1647};
1648
1649typedef blk_status_t (integrity_processing_fn) (struct blk_integrity_iter *);
1650typedef void (integrity_prepare_fn) (struct request *);
1651typedef void (integrity_complete_fn) (struct request *, unsigned int);
1652
1653struct blk_integrity_profile {
1654 integrity_processing_fn *generate_fn;
1655 integrity_processing_fn *verify_fn;
1656 integrity_prepare_fn *prepare_fn;
1657 integrity_complete_fn *complete_fn;
1658 const char *name;
1659};
1660
1661extern void blk_integrity_register(struct gendisk *, struct blk_integrity *);
1662extern void blk_integrity_unregister(struct gendisk *);
1663extern int blk_integrity_compare(struct gendisk *, struct gendisk *);
1664extern int blk_rq_map_integrity_sg(struct request_queue *, struct bio *,
1665 struct scatterlist *);
1666extern int blk_rq_count_integrity_sg(struct request_queue *, struct bio *);
1667
1668static inline struct blk_integrity *blk_get_integrity(struct gendisk *disk)
1669{
1670 struct blk_integrity *bi = &disk->queue->integrity;
1671
1672 if (!bi->profile)
1673 return NULL;
1674
1675 return bi;
1676}
1677
1678static inline
1679struct blk_integrity *bdev_get_integrity(struct block_device *bdev)
1680{
1681 return blk_get_integrity(bdev->bd_disk);
1682}
1683
1684static inline bool
1685blk_integrity_queue_supports_integrity(struct request_queue *q)
1686{
1687 return q->integrity.profile;
1688}
1689
1690static inline bool blk_integrity_rq(struct request *rq)
1691{
1692 return rq->cmd_flags & REQ_INTEGRITY;
1693}
1694
1695static inline void blk_queue_max_integrity_segments(struct request_queue *q,
1696 unsigned int segs)
1697{
1698 q->limits.max_integrity_segments = segs;
1699}
1700
1701static inline unsigned short
1702queue_max_integrity_segments(const struct request_queue *q)
1703{
1704 return q->limits.max_integrity_segments;
1705}
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717static inline unsigned int bio_integrity_intervals(struct blk_integrity *bi,
1718 unsigned int sectors)
1719{
1720 return sectors >> (bi->interval_exp - 9);
1721}
1722
1723static inline unsigned int bio_integrity_bytes(struct blk_integrity *bi,
1724 unsigned int sectors)
1725{
1726 return bio_integrity_intervals(bi, sectors) * bi->tuple_size;
1727}
1728
1729
1730
1731
1732
1733static inline struct bio_vec *rq_integrity_vec(struct request *rq)
1734{
1735 if (WARN_ON_ONCE(queue_max_integrity_segments(rq->q) > 1))
1736 return NULL;
1737 return rq->bio->bi_integrity->bip_vec;
1738}
1739
1740#else
1741
1742struct bio;
1743struct block_device;
1744struct gendisk;
1745struct blk_integrity;
1746
1747static inline int blk_integrity_rq(struct request *rq)
1748{
1749 return 0;
1750}
1751static inline int blk_rq_count_integrity_sg(struct request_queue *q,
1752 struct bio *b)
1753{
1754 return 0;
1755}
1756static inline int blk_rq_map_integrity_sg(struct request_queue *q,
1757 struct bio *b,
1758 struct scatterlist *s)
1759{
1760 return 0;
1761}
1762static inline struct blk_integrity *bdev_get_integrity(struct block_device *b)
1763{
1764 return NULL;
1765}
1766static inline struct blk_integrity *blk_get_integrity(struct gendisk *disk)
1767{
1768 return NULL;
1769}
1770static inline bool
1771blk_integrity_queue_supports_integrity(struct request_queue *q)
1772{
1773 return false;
1774}
1775static inline int blk_integrity_compare(struct gendisk *a, struct gendisk *b)
1776{
1777 return 0;
1778}
1779static inline void blk_integrity_register(struct gendisk *d,
1780 struct blk_integrity *b)
1781{
1782}
1783static inline void blk_integrity_unregister(struct gendisk *d)
1784{
1785}
1786static inline void blk_queue_max_integrity_segments(struct request_queue *q,
1787 unsigned int segs)
1788{
1789}
1790static inline unsigned short queue_max_integrity_segments(const struct request_queue *q)
1791{
1792 return 0;
1793}
1794
1795static inline unsigned int bio_integrity_intervals(struct blk_integrity *bi,
1796 unsigned int sectors)
1797{
1798 return 0;
1799}
1800
1801static inline unsigned int bio_integrity_bytes(struct blk_integrity *bi,
1802 unsigned int sectors)
1803{
1804 return 0;
1805}
1806
1807static inline struct bio_vec *rq_integrity_vec(struct request *rq)
1808{
1809 return NULL;
1810}
1811
1812#endif
1813
1814#ifdef CONFIG_BLK_INLINE_ENCRYPTION
1815
1816bool blk_ksm_register(struct blk_keyslot_manager *ksm, struct request_queue *q);
1817
1818void blk_ksm_unregister(struct request_queue *q);
1819
1820#else
1821
1822static inline bool blk_ksm_register(struct blk_keyslot_manager *ksm,
1823 struct request_queue *q)
1824{
1825 return true;
1826}
1827
1828static inline void blk_ksm_unregister(struct request_queue *q) { }
1829
1830#endif
1831
1832
1833struct block_device_operations {
1834 blk_qc_t (*submit_bio) (struct bio *bio);
1835 int (*open) (struct block_device *, fmode_t);
1836 void (*release) (struct gendisk *, fmode_t);
1837 int (*rw_page)(struct block_device *, sector_t, struct page *, unsigned int);
1838 int (*ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
1839 int (*compat_ioctl) (struct block_device *, fmode_t, unsigned, unsigned long);
1840 unsigned int (*check_events) (struct gendisk *disk,
1841 unsigned int clearing);
1842 void (*unlock_native_capacity) (struct gendisk *);
1843 int (*getgeo)(struct block_device *, struct hd_geometry *);
1844 int (*set_read_only)(struct block_device *bdev, bool ro);
1845
1846 void (*swap_slot_free_notify) (struct block_device *, unsigned long);
1847 int (*report_zones)(struct gendisk *, sector_t sector,
1848 unsigned int nr_zones, report_zones_cb cb, void *data);
1849 char *(*devnode)(struct gendisk *disk, umode_t *mode);
1850 struct module *owner;
1851 const struct pr_ops *pr_ops;
1852
1853
1854
1855
1856
1857
1858 int (*alternative_gpt_sector)(struct gendisk *disk, sector_t *sector);
1859};
1860
1861#ifdef CONFIG_COMPAT
1862extern int blkdev_compat_ptr_ioctl(struct block_device *, fmode_t,
1863 unsigned int, unsigned long);
1864#else
1865#define blkdev_compat_ptr_ioctl NULL
1866#endif
1867
1868extern int bdev_read_page(struct block_device *, sector_t, struct page *);
1869extern int bdev_write_page(struct block_device *, sector_t, struct page *,
1870 struct writeback_control *);
1871
1872#ifdef CONFIG_BLK_DEV_ZONED
1873bool blk_req_needs_zone_write_lock(struct request *rq);
1874bool blk_req_zone_write_trylock(struct request *rq);
1875void __blk_req_zone_write_lock(struct request *rq);
1876void __blk_req_zone_write_unlock(struct request *rq);
1877
1878static inline void blk_req_zone_write_lock(struct request *rq)
1879{
1880 if (blk_req_needs_zone_write_lock(rq))
1881 __blk_req_zone_write_lock(rq);
1882}
1883
1884static inline void blk_req_zone_write_unlock(struct request *rq)
1885{
1886 if (rq->rq_flags & RQF_ZONE_WRITE_LOCKED)
1887 __blk_req_zone_write_unlock(rq);
1888}
1889
1890static inline bool blk_req_zone_is_write_locked(struct request *rq)
1891{
1892 return rq->q->seq_zones_wlock &&
1893 test_bit(blk_rq_zone_no(rq), rq->q->seq_zones_wlock);
1894}
1895
1896static inline bool blk_req_can_dispatch_to_zone(struct request *rq)
1897{
1898 if (!blk_req_needs_zone_write_lock(rq))
1899 return true;
1900 return !blk_req_zone_is_write_locked(rq);
1901}
1902#else
1903static inline bool blk_req_needs_zone_write_lock(struct request *rq)
1904{
1905 return false;
1906}
1907
1908static inline void blk_req_zone_write_lock(struct request *rq)
1909{
1910}
1911
1912static inline void blk_req_zone_write_unlock(struct request *rq)
1913{
1914}
1915static inline bool blk_req_zone_is_write_locked(struct request *rq)
1916{
1917 return false;
1918}
1919
1920static inline bool blk_req_can_dispatch_to_zone(struct request *rq)
1921{
1922 return true;
1923}
1924#endif
1925
1926static inline void blk_wake_io_task(struct task_struct *waiter)
1927{
1928
1929
1930
1931
1932
1933 if (waiter == current)
1934 __set_current_state(TASK_RUNNING);
1935 else
1936 wake_up_process(waiter);
1937}
1938
1939unsigned long disk_start_io_acct(struct gendisk *disk, unsigned int sectors,
1940 unsigned int op);
1941void disk_end_io_acct(struct gendisk *disk, unsigned int op,
1942 unsigned long start_time);
1943
1944unsigned long bio_start_io_acct(struct bio *bio);
1945void bio_end_io_acct_remapped(struct bio *bio, unsigned long start_time,
1946 struct block_device *orig_bdev);
1947
1948
1949
1950
1951
1952
1953static inline void bio_end_io_acct(struct bio *bio, unsigned long start_time)
1954{
1955 return bio_end_io_acct_remapped(bio, start_time, bio->bi_bdev);
1956}
1957
1958int bdev_read_only(struct block_device *bdev);
1959int set_blocksize(struct block_device *bdev, int size);
1960
1961const char *bdevname(struct block_device *bdev, char *buffer);
1962int lookup_bdev(const char *pathname, dev_t *dev);
1963
1964void blkdev_show(struct seq_file *seqf, off_t offset);
1965
1966#define BDEVNAME_SIZE 32
1967#define BDEVT_SIZE 10
1968#ifdef CONFIG_BLOCK
1969#define BLKDEV_MAJOR_MAX 512
1970#else
1971#define BLKDEV_MAJOR_MAX 0
1972#endif
1973
1974struct block_device *blkdev_get_by_path(const char *path, fmode_t mode,
1975 void *holder);
1976struct block_device *blkdev_get_by_dev(dev_t dev, fmode_t mode, void *holder);
1977int bd_prepare_to_claim(struct block_device *bdev, void *holder);
1978void bd_abort_claiming(struct block_device *bdev, void *holder);
1979void blkdev_put(struct block_device *bdev, fmode_t mode);
1980
1981
1982struct block_device *blkdev_get_no_open(dev_t dev);
1983void blkdev_put_no_open(struct block_device *bdev);
1984
1985struct block_device *bdev_alloc(struct gendisk *disk, u8 partno);
1986void bdev_add(struct block_device *bdev, dev_t dev);
1987struct block_device *I_BDEV(struct inode *inode);
1988int truncate_bdev_range(struct block_device *bdev, fmode_t mode, loff_t lstart,
1989 loff_t lend);
1990
1991#ifdef CONFIG_BLOCK
1992void invalidate_bdev(struct block_device *bdev);
1993int sync_blockdev(struct block_device *bdev);
1994#else
1995static inline void invalidate_bdev(struct block_device *bdev)
1996{
1997}
1998static inline int sync_blockdev(struct block_device *bdev)
1999{
2000 return 0;
2001}
2002#endif
2003int fsync_bdev(struct block_device *bdev);
2004
2005int freeze_bdev(struct block_device *bdev);
2006int thaw_bdev(struct block_device *bdev);
2007
2008#endif
2009