1
2#ifndef BLK_INTERNAL_H
3#define BLK_INTERNAL_H
4
5#include <linux/idr.h>
6#include <linux/blk-mq.h>
7#include "blk-mq.h"
8
9
10#define BLK_BATCH_TIME (HZ/50UL)
11
12
13#define BLK_BATCH_REQ 32
14
15
16#define BLK_MAX_TIMEOUT (5 * HZ)
17
18#ifdef CONFIG_DEBUG_FS
19extern struct dentry *blk_debugfs_root;
20#endif
21
22struct blk_flush_queue {
23 unsigned int flush_queue_delayed:1;
24 unsigned int flush_pending_idx:1;
25 unsigned int flush_running_idx:1;
26 unsigned long flush_pending_since;
27 struct list_head flush_queue[2];
28 struct list_head flush_data_in_flight;
29 struct request *flush_rq;
30
31
32
33
34
35 struct request *orig_rq;
36 spinlock_t mq_flush_lock;
37};
38
39extern struct kmem_cache *blk_requestq_cachep;
40extern struct kmem_cache *request_cachep;
41extern struct kobj_type blk_queue_ktype;
42extern struct ida blk_queue_ida;
43
44static inline struct blk_flush_queue *blk_get_flush_queue(
45 struct request_queue *q, struct blk_mq_ctx *ctx)
46{
47 if (q->mq_ops)
48 return blk_mq_map_queue(q, ctx->cpu)->fq;
49 return q->fq;
50}
51
52static inline void __blk_get_queue(struct request_queue *q)
53{
54 kobject_get(&q->kobj);
55}
56
57struct blk_flush_queue *blk_alloc_flush_queue(struct request_queue *q,
58 int node, int cmd_size);
59void blk_free_flush_queue(struct blk_flush_queue *q);
60
61int blk_init_rl(struct request_list *rl, struct request_queue *q,
62 gfp_t gfp_mask);
63void blk_exit_rl(struct request_queue *q, struct request_list *rl);
64void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
65 struct bio *bio);
66void blk_queue_bypass_start(struct request_queue *q);
67void blk_queue_bypass_end(struct request_queue *q);
68void __blk_queue_free_tags(struct request_queue *q);
69void blk_freeze_queue(struct request_queue *q);
70
71static inline void blk_queue_enter_live(struct request_queue *q)
72{
73
74
75
76
77
78
79 percpu_ref_get(&q->q_usage_counter);
80}
81
82#ifdef CONFIG_BLK_DEV_INTEGRITY
83void blk_flush_integrity(void);
84bool __bio_integrity_endio(struct bio *);
85static inline bool bio_integrity_endio(struct bio *bio)
86{
87 if (bio_integrity(bio))
88 return __bio_integrity_endio(bio);
89 return true;
90}
91#else
92static inline void blk_flush_integrity(void)
93{
94}
95static inline bool bio_integrity_endio(struct bio *bio)
96{
97 return true;
98}
99#endif
100
101void blk_timeout_work(struct work_struct *work);
102unsigned long blk_rq_timeout(unsigned long timeout);
103void blk_add_timer(struct request *req);
104void blk_delete_timer(struct request *);
105
106
107bool bio_attempt_front_merge(struct request_queue *q, struct request *req,
108 struct bio *bio);
109bool bio_attempt_back_merge(struct request_queue *q, struct request *req,
110 struct bio *bio);
111bool bio_attempt_discard_merge(struct request_queue *q, struct request *req,
112 struct bio *bio);
113bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
114 unsigned int *request_count,
115 struct request **same_queue_rq);
116unsigned int blk_plug_queued_count(struct request_queue *q);
117
118void blk_account_io_start(struct request *req, bool new_io);
119void blk_account_io_completion(struct request *req, unsigned int bytes);
120void blk_account_io_done(struct request *req);
121
122
123
124
125enum rq_atomic_flags {
126 REQ_ATOM_COMPLETE = 0,
127 REQ_ATOM_STARTED,
128 REQ_ATOM_POLL_SLEPT,
129};
130
131
132
133
134
135static inline int blk_mark_rq_complete(struct request *rq)
136{
137 return test_and_set_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags);
138}
139
140static inline void blk_clear_rq_complete(struct request *rq)
141{
142 clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags);
143}
144
145
146
147
148#define ELV_ON_HASH(rq) ((rq)->rq_flags & RQF_HASHED)
149
150void blk_insert_flush(struct request *rq);
151
152static inline struct request *__elv_next_request(struct request_queue *q)
153{
154 struct request *rq;
155 struct blk_flush_queue *fq = blk_get_flush_queue(q, NULL);
156
157 WARN_ON_ONCE(q->mq_ops);
158
159 while (1) {
160 if (!list_empty(&q->queue_head)) {
161 rq = list_entry_rq(q->queue_head.next);
162 return rq;
163 }
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180 if (fq->flush_pending_idx != fq->flush_running_idx &&
181 !queue_flush_queueable(q)) {
182 fq->flush_queue_delayed = 1;
183 return NULL;
184 }
185 if (unlikely(blk_queue_bypass(q)) ||
186 !q->elevator->type->ops.sq.elevator_dispatch_fn(q, 0))
187 return NULL;
188 }
189}
190
191static inline void elv_activate_rq(struct request_queue *q, struct request *rq)
192{
193 struct elevator_queue *e = q->elevator;
194
195 if (e->type->ops.sq.elevator_activate_req_fn)
196 e->type->ops.sq.elevator_activate_req_fn(q, rq);
197}
198
199static inline void elv_deactivate_rq(struct request_queue *q, struct request *rq)
200{
201 struct elevator_queue *e = q->elevator;
202
203 if (e->type->ops.sq.elevator_deactivate_req_fn)
204 e->type->ops.sq.elevator_deactivate_req_fn(q, rq);
205}
206
207struct hd_struct *__disk_get_part(struct gendisk *disk, int partno);
208
209#ifdef CONFIG_FAIL_IO_TIMEOUT
210int blk_should_fake_timeout(struct request_queue *);
211ssize_t part_timeout_show(struct device *, struct device_attribute *, char *);
212ssize_t part_timeout_store(struct device *, struct device_attribute *,
213 const char *, size_t);
214#else
215static inline int blk_should_fake_timeout(struct request_queue *q)
216{
217 return 0;
218}
219#endif
220
221int ll_back_merge_fn(struct request_queue *q, struct request *req,
222 struct bio *bio);
223int ll_front_merge_fn(struct request_queue *q, struct request *req,
224 struct bio *bio);
225struct request *attempt_back_merge(struct request_queue *q, struct request *rq);
226struct request *attempt_front_merge(struct request_queue *q, struct request *rq);
227int blk_attempt_req_merge(struct request_queue *q, struct request *rq,
228 struct request *next);
229void blk_recalc_rq_segments(struct request *rq);
230void blk_rq_set_mixed_merge(struct request *rq);
231bool blk_rq_merge_ok(struct request *rq, struct bio *bio);
232enum elv_merge blk_try_merge(struct request *rq, struct bio *bio);
233
234void blk_queue_congestion_threshold(struct request_queue *q);
235
236int blk_dev_init(void);
237
238
239
240
241
242
243
244static inline int queue_congestion_on_threshold(struct request_queue *q)
245{
246 return q->nr_congestion_on;
247}
248
249
250
251
252static inline int queue_congestion_off_threshold(struct request_queue *q)
253{
254 return q->nr_congestion_off;
255}
256
257extern int blk_update_nr_requests(struct request_queue *, unsigned int);
258
259
260
261
262
263
264
265
266static inline int blk_do_io_stat(struct request *rq)
267{
268 return rq->rq_disk &&
269 (rq->rq_flags & RQF_IO_STAT) &&
270 !blk_rq_is_passthrough(rq);
271}
272
273static inline void req_set_nomerge(struct request_queue *q, struct request *req)
274{
275 req->cmd_flags |= REQ_NOMERGE;
276 if (req == q->last_merge)
277 q->last_merge = NULL;
278}
279
280
281
282
283void get_io_context(struct io_context *ioc);
284struct io_cq *ioc_lookup_icq(struct io_context *ioc, struct request_queue *q);
285struct io_cq *ioc_create_icq(struct io_context *ioc, struct request_queue *q,
286 gfp_t gfp_mask);
287void ioc_clear_queue(struct request_queue *q);
288
289int create_task_io_context(struct task_struct *task, gfp_t gfp_mask, int node);
290
291
292
293
294
295
296
297
298static inline struct io_context *rq_ioc(struct bio *bio)
299{
300#ifdef CONFIG_BLK_CGROUP
301 if (bio && bio->bi_ioc)
302 return bio->bi_ioc;
303#endif
304 return current->io_context;
305}
306
307
308
309
310
311
312
313
314
315
316
317
318
319static inline struct io_context *create_io_context(gfp_t gfp_mask, int node)
320{
321 WARN_ON_ONCE(irqs_disabled());
322 if (unlikely(!current->io_context))
323 create_task_io_context(current, gfp_mask, node);
324 return current->io_context;
325}
326
327
328
329
330#ifdef CONFIG_BLK_DEV_THROTTLING
331extern void blk_throtl_drain(struct request_queue *q);
332extern int blk_throtl_init(struct request_queue *q);
333extern void blk_throtl_exit(struct request_queue *q);
334extern void blk_throtl_register_queue(struct request_queue *q);
335#else
336static inline void blk_throtl_drain(struct request_queue *q) { }
337static inline int blk_throtl_init(struct request_queue *q) { return 0; }
338static inline void blk_throtl_exit(struct request_queue *q) { }
339static inline void blk_throtl_register_queue(struct request_queue *q) { }
340#endif
341#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
342extern ssize_t blk_throtl_sample_time_show(struct request_queue *q, char *page);
343extern ssize_t blk_throtl_sample_time_store(struct request_queue *q,
344 const char *page, size_t count);
345extern void blk_throtl_bio_endio(struct bio *bio);
346extern void blk_throtl_stat_add(struct request *rq, u64 time);
347#else
348static inline void blk_throtl_bio_endio(struct bio *bio) { }
349static inline void blk_throtl_stat_add(struct request *rq, u64 time) { }
350#endif
351
352#ifdef CONFIG_BOUNCE
353extern int init_emergency_isa_pool(void);
354extern void blk_queue_bounce(struct request_queue *q, struct bio **bio);
355#else
356static inline int init_emergency_isa_pool(void)
357{
358 return 0;
359}
360static inline void blk_queue_bounce(struct request_queue *q, struct bio **bio)
361{
362}
363#endif
364
365#endif
366