1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66#include <linux/kernel.h>
67#include <linux/module.h>
68#include <linux/bio.h>
69#include <linux/blkdev.h>
70#include <linux/gfp.h>
71#include <linux/blk-mq.h>
72
73#include "blk.h"
74#include "blk-mq.h"
75#include "blk-mq-tag.h"
76#include "blk-mq-sched.h"
77
78
79enum {
80 REQ_FSEQ_PREFLUSH = (1 << 0),
81 REQ_FSEQ_DATA = (1 << 1),
82 REQ_FSEQ_POSTFLUSH = (1 << 2),
83 REQ_FSEQ_DONE = (1 << 3),
84
85 REQ_FSEQ_ACTIONS = REQ_FSEQ_PREFLUSH | REQ_FSEQ_DATA |
86 REQ_FSEQ_POSTFLUSH,
87
88
89
90
91
92 FLUSH_PENDING_TIMEOUT = 5 * HZ,
93};
94
95static void blk_kick_flush(struct request_queue *q,
96 struct blk_flush_queue *fq, unsigned int flags);
97
98static unsigned int blk_flush_policy(unsigned long fflags, struct request *rq)
99{
100 unsigned int policy = 0;
101
102 if (blk_rq_sectors(rq))
103 policy |= REQ_FSEQ_DATA;
104
105 if (fflags & (1UL << QUEUE_FLAG_WC)) {
106 if (rq->cmd_flags & REQ_PREFLUSH)
107 policy |= REQ_FSEQ_PREFLUSH;
108 if (!(fflags & (1UL << QUEUE_FLAG_FUA)) &&
109 (rq->cmd_flags & REQ_FUA))
110 policy |= REQ_FSEQ_POSTFLUSH;
111 }
112 return policy;
113}
114
115static unsigned int blk_flush_cur_seq(struct request *rq)
116{
117 return 1 << ffz(rq->flush.seq);
118}
119
120static void blk_flush_restore_request(struct request *rq)
121{
122
123
124
125
126
127 rq->bio = rq->biotail;
128
129
130 rq->rq_flags &= ~RQF_FLUSH_SEQ;
131 rq->end_io = rq->flush.saved_end_io;
132}
133
134static void blk_flush_queue_rq(struct request *rq, bool add_front)
135{
136 blk_mq_add_to_requeue_list(rq, add_front, true);
137}
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155static void blk_flush_complete_seq(struct request *rq,
156 struct blk_flush_queue *fq,
157 unsigned int seq, blk_status_t error)
158{
159 struct request_queue *q = rq->q;
160 struct list_head *pending = &fq->flush_queue[fq->flush_pending_idx];
161 unsigned int cmd_flags;
162
163 BUG_ON(rq->flush.seq & seq);
164 rq->flush.seq |= seq;
165 cmd_flags = rq->cmd_flags;
166
167 if (likely(!error))
168 seq = blk_flush_cur_seq(rq);
169 else
170 seq = REQ_FSEQ_DONE;
171
172 switch (seq) {
173 case REQ_FSEQ_PREFLUSH:
174 case REQ_FSEQ_POSTFLUSH:
175
176 if (list_empty(pending))
177 fq->flush_pending_since = jiffies;
178 list_move_tail(&rq->flush.list, pending);
179 break;
180
181 case REQ_FSEQ_DATA:
182 list_move_tail(&rq->flush.list, &fq->flush_data_in_flight);
183 blk_flush_queue_rq(rq, true);
184 break;
185
186 case REQ_FSEQ_DONE:
187
188
189
190
191
192
193 BUG_ON(!list_empty(&rq->queuelist));
194 list_del_init(&rq->flush.list);
195 blk_flush_restore_request(rq);
196 blk_mq_end_request(rq, error);
197 break;
198
199 default:
200 BUG();
201 }
202
203 blk_kick_flush(q, fq, cmd_flags);
204}
205
206static void flush_end_io(struct request *flush_rq, blk_status_t error)
207{
208 struct request_queue *q = flush_rq->q;
209 struct list_head *running;
210 struct request *rq, *n;
211 unsigned long flags = 0;
212 struct blk_flush_queue *fq = blk_get_flush_queue(q, flush_rq->mq_ctx);
213 struct blk_mq_hw_ctx *hctx;
214
215
216 spin_lock_irqsave(&fq->mq_flush_lock, flags);
217 hctx = flush_rq->mq_hctx;
218 if (!q->elevator) {
219 blk_mq_tag_set_rq(hctx, flush_rq->tag, fq->orig_rq);
220 flush_rq->tag = -1;
221 } else {
222 blk_mq_put_driver_tag(flush_rq);
223 flush_rq->internal_tag = -1;
224 }
225
226 running = &fq->flush_queue[fq->flush_running_idx];
227 BUG_ON(fq->flush_pending_idx == fq->flush_running_idx);
228
229
230 fq->flush_running_idx ^= 1;
231
232
233 list_for_each_entry_safe(rq, n, running, flush.list) {
234 unsigned int seq = blk_flush_cur_seq(rq);
235
236 BUG_ON(seq != REQ_FSEQ_PREFLUSH && seq != REQ_FSEQ_POSTFLUSH);
237 blk_flush_complete_seq(rq, fq, seq, error);
238 }
239
240 fq->flush_queue_delayed = 0;
241 spin_unlock_irqrestore(&fq->mq_flush_lock, flags);
242}
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257static void blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq,
258 unsigned int flags)
259{
260 struct list_head *pending = &fq->flush_queue[fq->flush_pending_idx];
261 struct request *first_rq =
262 list_first_entry(pending, struct request, flush.list);
263 struct request *flush_rq = fq->flush_rq;
264
265
266 if (fq->flush_pending_idx != fq->flush_running_idx || list_empty(pending))
267 return;
268
269
270
271
272
273
274
275 if (!list_empty(&fq->flush_data_in_flight) && q->elevator &&
276 time_before(jiffies,
277 fq->flush_pending_since + FLUSH_PENDING_TIMEOUT))
278 return;
279
280
281
282
283
284 fq->flush_pending_idx ^= 1;
285
286 blk_rq_init(q, flush_rq);
287
288
289
290
291
292
293
294
295
296 flush_rq->mq_ctx = first_rq->mq_ctx;
297 flush_rq->mq_hctx = first_rq->mq_hctx;
298
299 if (!q->elevator) {
300 fq->orig_rq = first_rq;
301 flush_rq->tag = first_rq->tag;
302 blk_mq_tag_set_rq(flush_rq->mq_hctx, first_rq->tag, flush_rq);
303 } else {
304 flush_rq->internal_tag = first_rq->internal_tag;
305 }
306
307 flush_rq->cmd_flags = REQ_OP_FLUSH | REQ_PREFLUSH;
308 flush_rq->cmd_flags |= (flags & REQ_DRV) | (flags & REQ_FAILFAST_MASK);
309 flush_rq->rq_flags |= RQF_FLUSH_SEQ;
310 flush_rq->rq_disk = first_rq->rq_disk;
311 flush_rq->end_io = flush_end_io;
312
313 blk_flush_queue_rq(flush_rq, false);
314}
315
316static void mq_flush_data_end_io(struct request *rq, blk_status_t error)
317{
318 struct request_queue *q = rq->q;
319 struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
320 struct blk_mq_ctx *ctx = rq->mq_ctx;
321 unsigned long flags;
322 struct blk_flush_queue *fq = blk_get_flush_queue(q, ctx);
323
324 if (q->elevator) {
325 WARN_ON(rq->tag < 0);
326 blk_mq_put_driver_tag(rq);
327 }
328
329
330
331
332
333 spin_lock_irqsave(&fq->mq_flush_lock, flags);
334 blk_flush_complete_seq(rq, fq, REQ_FSEQ_DATA, error);
335 spin_unlock_irqrestore(&fq->mq_flush_lock, flags);
336
337 blk_mq_sched_restart(hctx);
338}
339
340
341
342
343
344
345
346
347
348
349void blk_insert_flush(struct request *rq)
350{
351 struct request_queue *q = rq->q;
352 unsigned long fflags = q->queue_flags;
353 unsigned int policy = blk_flush_policy(fflags, rq);
354 struct blk_flush_queue *fq = blk_get_flush_queue(q, rq->mq_ctx);
355
356
357
358
359
360 rq->cmd_flags &= ~REQ_PREFLUSH;
361 if (!(fflags & (1UL << QUEUE_FLAG_FUA)))
362 rq->cmd_flags &= ~REQ_FUA;
363
364
365
366
367
368
369 rq->cmd_flags |= REQ_SYNC;
370
371
372
373
374
375
376
377 if (!policy) {
378 blk_mq_end_request(rq, 0);
379 return;
380 }
381
382 BUG_ON(rq->bio != rq->biotail);
383
384
385
386
387
388
389 if ((policy & REQ_FSEQ_DATA) &&
390 !(policy & (REQ_FSEQ_PREFLUSH | REQ_FSEQ_POSTFLUSH))) {
391 blk_mq_request_bypass_insert(rq, false);
392 return;
393 }
394
395
396
397
398
399 memset(&rq->flush, 0, sizeof(rq->flush));
400 INIT_LIST_HEAD(&rq->flush.list);
401 rq->rq_flags |= RQF_FLUSH_SEQ;
402 rq->flush.saved_end_io = rq->end_io;
403
404 rq->end_io = mq_flush_data_end_io;
405
406 spin_lock_irq(&fq->mq_flush_lock);
407 blk_flush_complete_seq(rq, fq, REQ_FSEQ_ACTIONS & ~policy, 0);
408 spin_unlock_irq(&fq->mq_flush_lock);
409}
410
411
412
413
414
415
416
417
418
419
420
421
422int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask,
423 sector_t *error_sector)
424{
425 struct request_queue *q;
426 struct bio *bio;
427 int ret = 0;
428
429 if (bdev->bd_disk == NULL)
430 return -ENXIO;
431
432 q = bdev_get_queue(bdev);
433 if (!q)
434 return -ENXIO;
435
436
437
438
439
440
441
442 if (!q->make_request_fn)
443 return -ENXIO;
444
445 bio = bio_alloc(gfp_mask, 0);
446 bio_set_dev(bio, bdev);
447 bio->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
448
449 ret = submit_bio_wait(bio);
450
451
452
453
454
455
456 if (error_sector)
457 *error_sector = bio->bi_iter.bi_sector;
458
459 bio_put(bio);
460 return ret;
461}
462EXPORT_SYMBOL(blkdev_issue_flush);
463
464struct blk_flush_queue *blk_alloc_flush_queue(struct request_queue *q,
465 int node, int cmd_size, gfp_t flags)
466{
467 struct blk_flush_queue *fq;
468 int rq_sz = sizeof(struct request);
469
470 fq = kzalloc_node(sizeof(*fq), flags, node);
471 if (!fq)
472 goto fail;
473
474 spin_lock_init(&fq->mq_flush_lock);
475
476 rq_sz = round_up(rq_sz + cmd_size, cache_line_size());
477 fq->flush_rq = kzalloc_node(rq_sz, flags, node);
478 if (!fq->flush_rq)
479 goto fail_rq;
480
481 INIT_LIST_HEAD(&fq->flush_queue[0]);
482 INIT_LIST_HEAD(&fq->flush_queue[1]);
483 INIT_LIST_HEAD(&fq->flush_data_in_flight);
484
485 return fq;
486
487 fail_rq:
488 kfree(fq);
489 fail:
490 return NULL;
491}
492
493void blk_free_flush_queue(struct blk_flush_queue *fq)
494{
495
496 if (!fq)
497 return;
498
499 kfree(fq->flush_rq);
500 kfree(fq);
501}
502