1
2#ifndef BLK_MQ_H
3#define BLK_MQ_H
4
5#include <linux/blkdev.h>
6#include <linux/sbitmap.h>
7#include <linux/srcu.h>
8
9struct blk_mq_tags;
10struct blk_flush_queue;
11
12
13
14
15
16struct blk_mq_hw_ctx {
17 struct {
18
19 spinlock_t lock;
20
21
22
23
24
25
26
27 struct list_head dispatch;
28
29
30
31
32 unsigned long state;
33 } ____cacheline_aligned_in_smp;
34
35
36
37
38 struct delayed_work run_work;
39
40 cpumask_var_t cpumask;
41
42
43
44
45 int next_cpu;
46
47
48
49
50 int next_cpu_batch;
51
52
53 unsigned long flags;
54
55
56
57
58
59 void *sched_data;
60
61
62
63 struct request_queue *queue;
64
65 struct blk_flush_queue *fq;
66
67
68
69
70
71 void *driver_data;
72
73
74
75
76
77 struct sbitmap ctx_map;
78
79
80
81
82
83 struct blk_mq_ctx *dispatch_from;
84
85
86
87
88
89 unsigned int dispatch_busy;
90
91
92 unsigned short type;
93
94 unsigned short nr_ctx;
95
96 struct blk_mq_ctx **ctxs;
97
98
99 spinlock_t dispatch_wait_lock;
100
101
102
103
104 wait_queue_entry_t dispatch_wait;
105
106
107
108
109
110 atomic_t wait_index;
111
112
113
114
115
116 struct blk_mq_tags *tags;
117
118
119
120
121
122 struct blk_mq_tags *sched_tags;
123
124
125 unsigned long queued;
126
127 unsigned long run;
128#define BLK_MQ_MAX_DISPATCH_ORDER 7
129
130 unsigned long dispatched[BLK_MQ_MAX_DISPATCH_ORDER];
131
132
133 unsigned int numa_node;
134
135 unsigned int queue_num;
136
137
138
139
140
141 atomic_t nr_active;
142
143
144 struct hlist_node cpuhp_online;
145
146 struct hlist_node cpuhp_dead;
147
148 struct kobject kobj;
149
150
151 unsigned long poll_considered;
152
153 unsigned long poll_invoked;
154
155 unsigned long poll_success;
156
157#ifdef CONFIG_BLK_DEBUG_FS
158
159
160
161
162 struct dentry *debugfs_dir;
163
164 struct dentry *sched_debugfs_dir;
165#endif
166
167
168
169
170
171 struct list_head hctx_list;
172
173
174
175
176
177
178 struct srcu_struct srcu[];
179};
180
181
182
183
184
185
186
187
188
189
190
191struct blk_mq_queue_map {
192 unsigned int *mq_map;
193 unsigned int nr_queues;
194 unsigned int queue_offset;
195};
196
197
198
199
200
201
202
203
204enum hctx_type {
205 HCTX_TYPE_DEFAULT,
206 HCTX_TYPE_READ,
207 HCTX_TYPE_POLL,
208
209 HCTX_MAX_TYPES,
210};
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240struct blk_mq_tag_set {
241 struct blk_mq_queue_map map[HCTX_MAX_TYPES];
242 unsigned int nr_maps;
243 const struct blk_mq_ops *ops;
244 unsigned int nr_hw_queues;
245 unsigned int queue_depth;
246 unsigned int reserved_tags;
247 unsigned int cmd_size;
248 int numa_node;
249 unsigned int timeout;
250 unsigned int flags;
251 void *driver_data;
252
253 struct blk_mq_tags **tags;
254
255 struct mutex tag_list_lock;
256 struct list_head tag_list;
257};
258
259
260
261
262
263
264
265struct blk_mq_queue_data {
266 struct request *rq;
267 bool last;
268};
269
270typedef bool (busy_iter_fn)(struct blk_mq_hw_ctx *, struct request *, void *,
271 bool);
272typedef bool (busy_tag_iter_fn)(struct request *, void *, bool);
273
274
275
276
277
278struct blk_mq_ops {
279
280
281
282 blk_status_t (*queue_rq)(struct blk_mq_hw_ctx *,
283 const struct blk_mq_queue_data *);
284
285
286
287
288
289
290
291
292 void (*commit_rqs)(struct blk_mq_hw_ctx *);
293
294
295
296
297
298
299
300 bool (*get_budget)(struct request_queue *);
301
302
303
304
305 void (*put_budget)(struct request_queue *);
306
307
308
309
310 enum blk_eh_timer_return (*timeout)(struct request *, bool);
311
312
313
314
315 int (*poll)(struct blk_mq_hw_ctx *);
316
317
318
319
320 void (*complete)(struct request *);
321
322
323
324
325
326
327 int (*init_hctx)(struct blk_mq_hw_ctx *, void *, unsigned int);
328
329
330
331 void (*exit_hctx)(struct blk_mq_hw_ctx *, unsigned int);
332
333
334
335
336
337
338
339
340 int (*init_request)(struct blk_mq_tag_set *set, struct request *,
341 unsigned int, unsigned int);
342
343
344
345 void (*exit_request)(struct blk_mq_tag_set *set, struct request *,
346 unsigned int);
347
348
349
350
351 void (*initialize_rq_fn)(struct request *rq);
352
353
354
355
356
357 void (*cleanup_rq)(struct request *);
358
359
360
361
362 bool (*busy)(struct request_queue *);
363
364
365
366
367
368 int (*map_queues)(struct blk_mq_tag_set *set);
369
370#ifdef CONFIG_BLK_DEBUG_FS
371
372
373
374
375 void (*show_rq)(struct seq_file *m, struct request *rq);
376#endif
377};
378
379enum {
380 BLK_MQ_F_SHOULD_MERGE = 1 << 0,
381 BLK_MQ_F_TAG_SHARED = 1 << 1,
382
383
384
385
386 BLK_MQ_F_STACKING = 1 << 2,
387 BLK_MQ_F_BLOCKING = 1 << 5,
388 BLK_MQ_F_NO_SCHED = 1 << 6,
389 BLK_MQ_F_ALLOC_POLICY_START_BIT = 8,
390 BLK_MQ_F_ALLOC_POLICY_BITS = 1,
391
392 BLK_MQ_S_STOPPED = 0,
393 BLK_MQ_S_TAG_ACTIVE = 1,
394 BLK_MQ_S_SCHED_RESTART = 2,
395
396
397 BLK_MQ_S_INACTIVE = 3,
398
399 BLK_MQ_MAX_DEPTH = 10240,
400
401 BLK_MQ_CPU_WORK_BATCH = 8,
402};
403#define BLK_MQ_FLAG_TO_ALLOC_POLICY(flags) \
404 ((flags >> BLK_MQ_F_ALLOC_POLICY_START_BIT) & \
405 ((1 << BLK_MQ_F_ALLOC_POLICY_BITS) - 1))
406#define BLK_ALLOC_POLICY_TO_MQ_FLAG(policy) \
407 ((policy & ((1 << BLK_MQ_F_ALLOC_POLICY_BITS) - 1)) \
408 << BLK_MQ_F_ALLOC_POLICY_START_BIT)
409
410struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *);
411struct request_queue *blk_mq_init_queue_data(struct blk_mq_tag_set *set,
412 void *queuedata);
413struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
414 struct request_queue *q,
415 bool elevator_init);
416struct request_queue *blk_mq_init_sq_queue(struct blk_mq_tag_set *set,
417 const struct blk_mq_ops *ops,
418 unsigned int queue_depth,
419 unsigned int set_flags);
420void blk_mq_unregister_dev(struct device *, struct request_queue *);
421
422int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set);
423void blk_mq_free_tag_set(struct blk_mq_tag_set *set);
424
425void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule);
426
427void blk_mq_free_request(struct request *rq);
428
429bool blk_mq_queue_inflight(struct request_queue *q);
430
431enum {
432
433 BLK_MQ_REQ_NOWAIT = (__force blk_mq_req_flags_t)(1 << 0),
434
435 BLK_MQ_REQ_RESERVED = (__force blk_mq_req_flags_t)(1 << 1),
436
437 BLK_MQ_REQ_PREEMPT = (__force blk_mq_req_flags_t)(1 << 3),
438};
439
440struct request *blk_mq_alloc_request(struct request_queue *q, unsigned int op,
441 blk_mq_req_flags_t flags);
442struct request *blk_mq_alloc_request_hctx(struct request_queue *q,
443 unsigned int op, blk_mq_req_flags_t flags,
444 unsigned int hctx_idx);
445struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag);
446
447enum {
448 BLK_MQ_UNIQUE_TAG_BITS = 16,
449 BLK_MQ_UNIQUE_TAG_MASK = (1 << BLK_MQ_UNIQUE_TAG_BITS) - 1,
450};
451
452u32 blk_mq_unique_tag(struct request *rq);
453
454static inline u16 blk_mq_unique_tag_to_hwq(u32 unique_tag)
455{
456 return unique_tag >> BLK_MQ_UNIQUE_TAG_BITS;
457}
458
459static inline u16 blk_mq_unique_tag_to_tag(u32 unique_tag)
460{
461 return unique_tag & BLK_MQ_UNIQUE_TAG_MASK;
462}
463
464
465
466
467
468static inline enum mq_rq_state blk_mq_rq_state(struct request *rq)
469{
470 return READ_ONCE(rq->state);
471}
472
473static inline int blk_mq_request_started(struct request *rq)
474{
475 return blk_mq_rq_state(rq) != MQ_RQ_IDLE;
476}
477
478static inline int blk_mq_request_completed(struct request *rq)
479{
480 return blk_mq_rq_state(rq) == MQ_RQ_COMPLETE;
481}
482
483void blk_mq_start_request(struct request *rq);
484void blk_mq_end_request(struct request *rq, blk_status_t error);
485void __blk_mq_end_request(struct request *rq, blk_status_t error);
486
487void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list);
488void blk_mq_kick_requeue_list(struct request_queue *q);
489void blk_mq_delay_kick_requeue_list(struct request_queue *q, unsigned long msecs);
490void blk_mq_complete_request(struct request *rq);
491bool blk_mq_complete_request_remote(struct request *rq);
492bool blk_mq_bio_list_merge(struct request_queue *q, struct list_head *list,
493 struct bio *bio, unsigned int nr_segs);
494bool blk_mq_queue_stopped(struct request_queue *q);
495void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx);
496void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx);
497void blk_mq_stop_hw_queues(struct request_queue *q);
498void blk_mq_start_hw_queues(struct request_queue *q);
499void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async);
500void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async);
501void blk_mq_quiesce_queue(struct request_queue *q);
502void blk_mq_unquiesce_queue(struct request_queue *q);
503void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs);
504void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async);
505void blk_mq_run_hw_queues(struct request_queue *q, bool async);
506void blk_mq_delay_run_hw_queues(struct request_queue *q, unsigned long msecs);
507void blk_mq_tagset_busy_iter(struct blk_mq_tag_set *tagset,
508 busy_tag_iter_fn *fn, void *priv);
509void blk_mq_tagset_wait_completed_request(struct blk_mq_tag_set *tagset);
510void blk_mq_freeze_queue(struct request_queue *q);
511void blk_mq_unfreeze_queue(struct request_queue *q);
512void blk_freeze_queue_start(struct request_queue *q);
513void blk_mq_freeze_queue_wait(struct request_queue *q);
514int blk_mq_freeze_queue_wait_timeout(struct request_queue *q,
515 unsigned long timeout);
516
517int blk_mq_map_queues(struct blk_mq_queue_map *qmap);
518void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues);
519
520void blk_mq_quiesce_queue_nowait(struct request_queue *q);
521
522unsigned int blk_mq_rq_cpu(struct request *rq);
523
524bool __blk_should_fake_timeout(struct request_queue *q);
525static inline bool blk_should_fake_timeout(struct request_queue *q)
526{
527 if (IS_ENABLED(CONFIG_FAIL_IO_TIMEOUT) &&
528 test_bit(QUEUE_FLAG_FAIL_IO, &q->queue_flags))
529 return __blk_should_fake_timeout(q);
530 return false;
531}
532
533
534
535
536
537
538
539
540
541
542static inline struct request *blk_mq_rq_from_pdu(void *pdu)
543{
544 return pdu - sizeof(struct request);
545}
546
547
548
549
550
551
552
553
554
555
556static inline void *blk_mq_rq_to_pdu(struct request *rq)
557{
558 return rq + 1;
559}
560
561#define queue_for_each_hw_ctx(q, hctx, i) \
562 for ((i) = 0; (i) < (q)->nr_hw_queues && \
563 ({ hctx = (q)->queue_hw_ctx[i]; 1; }); (i)++)
564
565#define hctx_for_each_ctx(hctx, ctx, i) \
566 for ((i) = 0; (i) < (hctx)->nr_ctx && \
567 ({ ctx = (hctx)->ctxs[(i)]; 1; }); (i)++)
568
569static inline blk_qc_t request_to_qc_t(struct blk_mq_hw_ctx *hctx,
570 struct request *rq)
571{
572 if (rq->tag != -1)
573 return rq->tag | (hctx->queue_num << BLK_QC_T_SHIFT);
574
575 return rq->internal_tag | (hctx->queue_num << BLK_QC_T_SHIFT) |
576 BLK_QC_T_INTERNAL;
577}
578
579static inline void blk_mq_cleanup_rq(struct request *rq)
580{
581 if (rq->q->mq_ops->cleanup_rq)
582 rq->q->mq_ops->cleanup_rq(rq);
583}
584
585blk_qc_t blk_mq_submit_bio(struct bio *bio);
586
587#endif
588