1
2
3
4
5
6
7
8
9
10
11
12
13
14#include <linux/kernel.h>
15#include <linux/module.h>
16#include <linux/backing-dev.h>
17#include <linux/bio.h>
18#include <linux/blkdev.h>
19#include <linux/highmem.h>
20#include <linux/mm.h>
21#include <linux/kernel_stat.h>
22#include <linux/string.h>
23#include <linux/init.h>
24#include <linux/completion.h>
25#include <linux/slab.h>
26#include <linux/swap.h>
27#include <linux/writeback.h>
28#include <linux/task_io_accounting_ops.h>
29#include <linux/fault-inject.h>
30#include <linux/list_sort.h>
31#include <linux/delay.h>
32#include <linux/ratelimit.h>
33#include <linux/pm_runtime.h>
34
35#define CREATE_TRACE_POINTS
36#include <trace/events/block.h>
37
38#include "blk.h"
39#include "blk-cgroup.h"
40
41EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_remap);
42EXPORT_TRACEPOINT_SYMBOL_GPL(block_rq_remap);
43EXPORT_TRACEPOINT_SYMBOL_GPL(block_bio_complete);
44EXPORT_TRACEPOINT_SYMBOL_GPL(block_unplug);
45
46DEFINE_IDA(blk_queue_ida);
47
48
49
50
51static struct kmem_cache *request_cachep;
52
53
54
55
56struct kmem_cache *blk_requestq_cachep;
57
58
59
60
61static struct workqueue_struct *kblockd_workqueue;
62
63static void drive_stat_acct(struct request *rq, int new_io)
64{
65 struct hd_struct *part;
66 int rw = rq_data_dir(rq);
67 int cpu;
68
69 if (!blk_do_io_stat(rq))
70 return;
71
72 cpu = part_stat_lock();
73
74 if (!new_io) {
75 part = rq->part;
76 part_stat_inc(cpu, part, merges[rw]);
77 } else {
78 part = disk_map_sector_rcu(rq->rq_disk, blk_rq_pos(rq));
79 if (!hd_struct_try_get(part)) {
80
81
82
83
84
85
86
87
88 part = &rq->rq_disk->part0;
89 hd_struct_get(part);
90 }
91 part_round_stats(cpu, part);
92 part_inc_in_flight(part, rw);
93 rq->part = part;
94 }
95
96 part_stat_unlock();
97}
98
99void blk_queue_congestion_threshold(struct request_queue *q)
100{
101 int nr;
102
103 nr = q->nr_requests - (q->nr_requests / 8) + 1;
104 if (nr > q->nr_requests)
105 nr = q->nr_requests;
106 q->nr_congestion_on = nr;
107
108 nr = q->nr_requests - (q->nr_requests / 8) - (q->nr_requests / 16) - 1;
109 if (nr < 1)
110 nr = 1;
111 q->nr_congestion_off = nr;
112}
113
114
115
116
117
118
119
120
121
122
123struct backing_dev_info *blk_get_backing_dev_info(struct block_device *bdev)
124{
125 struct backing_dev_info *ret = NULL;
126 struct request_queue *q = bdev_get_queue(bdev);
127
128 if (q)
129 ret = &q->backing_dev_info;
130 return ret;
131}
132EXPORT_SYMBOL(blk_get_backing_dev_info);
133
134void blk_rq_init(struct request_queue *q, struct request *rq)
135{
136 memset(rq, 0, sizeof(*rq));
137
138 INIT_LIST_HEAD(&rq->queuelist);
139 INIT_LIST_HEAD(&rq->timeout_list);
140 rq->cpu = -1;
141 rq->q = q;
142 rq->__sector = (sector_t) -1;
143 INIT_HLIST_NODE(&rq->hash);
144 RB_CLEAR_NODE(&rq->rb_node);
145 rq->cmd = rq->__cmd;
146 rq->cmd_len = BLK_MAX_CDB;
147 rq->tag = -1;
148 rq->ref_count = 1;
149 rq->start_time = jiffies;
150 set_start_time_ns(rq);
151 rq->part = NULL;
152}
153EXPORT_SYMBOL(blk_rq_init);
154
155static void req_bio_endio(struct request *rq, struct bio *bio,
156 unsigned int nbytes, int error)
157{
158 if (error)
159 clear_bit(BIO_UPTODATE, &bio->bi_flags);
160 else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
161 error = -EIO;
162
163 if (unlikely(rq->cmd_flags & REQ_QUIET))
164 set_bit(BIO_QUIET, &bio->bi_flags);
165
166 bio_advance(bio, nbytes);
167
168
169 if (bio->bi_size == 0 && !(rq->cmd_flags & REQ_FLUSH_SEQ))
170 bio_endio(bio, error);
171}
172
173void blk_dump_rq_flags(struct request *rq, char *msg)
174{
175 int bit;
176
177 printk(KERN_INFO "%s: dev %s: type=%x, flags=%x\n", msg,
178 rq->rq_disk ? rq->rq_disk->disk_name : "?", rq->cmd_type,
179 rq->cmd_flags);
180
181 printk(KERN_INFO " sector %llu, nr/cnr %u/%u\n",
182 (unsigned long long)blk_rq_pos(rq),
183 blk_rq_sectors(rq), blk_rq_cur_sectors(rq));
184 printk(KERN_INFO " bio %p, biotail %p, buffer %p, len %u\n",
185 rq->bio, rq->biotail, rq->buffer, blk_rq_bytes(rq));
186
187 if (rq->cmd_type == REQ_TYPE_BLOCK_PC) {
188 printk(KERN_INFO " cdb: ");
189 for (bit = 0; bit < BLK_MAX_CDB; bit++)
190 printk("%02x ", rq->cmd[bit]);
191 printk("\n");
192 }
193}
194EXPORT_SYMBOL(blk_dump_rq_flags);
195
196static void blk_delay_work(struct work_struct *work)
197{
198 struct request_queue *q;
199
200 q = container_of(work, struct request_queue, delay_work.work);
201 spin_lock_irq(q->queue_lock);
202 __blk_run_queue(q);
203 spin_unlock_irq(q->queue_lock);
204}
205
206
207
208
209
210
211
212
213
214
215
216void blk_delay_queue(struct request_queue *q, unsigned long msecs)
217{
218 if (likely(!blk_queue_dead(q)))
219 queue_delayed_work(kblockd_workqueue, &q->delay_work,
220 msecs_to_jiffies(msecs));
221}
222EXPORT_SYMBOL(blk_delay_queue);
223
224
225
226
227
228
229
230
231
232
233void blk_start_queue(struct request_queue *q)
234{
235 WARN_ON(!irqs_disabled());
236
237 queue_flag_clear(QUEUE_FLAG_STOPPED, q);
238 __blk_run_queue(q);
239}
240EXPORT_SYMBOL(blk_start_queue);
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256void blk_stop_queue(struct request_queue *q)
257{
258 cancel_delayed_work(&q->delay_work);
259 queue_flag_set(QUEUE_FLAG_STOPPED, q);
260}
261EXPORT_SYMBOL(blk_stop_queue);
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281void blk_sync_queue(struct request_queue *q)
282{
283 del_timer_sync(&q->timeout);
284 cancel_delayed_work_sync(&q->delay_work);
285}
286EXPORT_SYMBOL(blk_sync_queue);
287
288
289
290
291
292
293
294
295
296
297
298
299inline void __blk_run_queue_uncond(struct request_queue *q)
300{
301 if (unlikely(blk_queue_dead(q)))
302 return;
303
304
305
306
307
308
309
310
311 q->request_fn_active++;
312 q->request_fn(q);
313 q->request_fn_active--;
314}
315
316
317
318
319
320
321
322
323
324void __blk_run_queue(struct request_queue *q)
325{
326 if (unlikely(blk_queue_stopped(q)))
327 return;
328
329 __blk_run_queue_uncond(q);
330}
331EXPORT_SYMBOL(__blk_run_queue);
332
333
334
335
336
337
338
339
340
341void blk_run_queue_async(struct request_queue *q)
342{
343 if (likely(!blk_queue_stopped(q) && !blk_queue_dead(q)))
344 mod_delayed_work(kblockd_workqueue, &q->delay_work, 0);
345}
346EXPORT_SYMBOL(blk_run_queue_async);
347
348
349
350
351
352
353
354
355
356void blk_run_queue(struct request_queue *q)
357{
358 unsigned long flags;
359
360 spin_lock_irqsave(q->queue_lock, flags);
361 __blk_run_queue(q);
362 spin_unlock_irqrestore(q->queue_lock, flags);
363}
364EXPORT_SYMBOL(blk_run_queue);
365
366void blk_put_queue(struct request_queue *q)
367{
368 kobject_put(&q->kobj);
369}
370EXPORT_SYMBOL(blk_put_queue);
371
372
373
374
375
376
377
378
379
380
381static void __blk_drain_queue(struct request_queue *q, bool drain_all)
382 __releases(q->queue_lock)
383 __acquires(q->queue_lock)
384{
385 int i;
386
387 lockdep_assert_held(q->queue_lock);
388
389 while (true) {
390 bool drain = false;
391
392
393
394
395
396 if (q->elevator)
397 elv_drain_elevator(q);
398
399 blkcg_drain_queue(q);
400
401
402
403
404
405
406
407
408 if (!list_empty(&q->queue_head) && q->request_fn)
409 __blk_run_queue(q);
410
411 drain |= q->nr_rqs_elvpriv;
412 drain |= q->request_fn_active;
413
414
415
416
417
418
419 if (drain_all) {
420 drain |= !list_empty(&q->queue_head);
421 for (i = 0; i < 2; i++) {
422 drain |= q->nr_rqs[i];
423 drain |= q->in_flight[i];
424 drain |= !list_empty(&q->flush_queue[i]);
425 }
426 }
427
428 if (!drain)
429 break;
430
431 spin_unlock_irq(q->queue_lock);
432
433 msleep(10);
434
435 spin_lock_irq(q->queue_lock);
436 }
437
438
439
440
441
442
443 if (q->request_fn) {
444 struct request_list *rl;
445
446 blk_queue_for_each_rl(rl, q)
447 for (i = 0; i < ARRAY_SIZE(rl->wait); i++)
448 wake_up_all(&rl->wait[i]);
449 }
450}
451
452
453
454
455
456
457
458
459
460
461
462void blk_queue_bypass_start(struct request_queue *q)
463{
464 bool drain;
465
466 spin_lock_irq(q->queue_lock);
467 drain = !q->bypass_depth++;
468 queue_flag_set(QUEUE_FLAG_BYPASS, q);
469 spin_unlock_irq(q->queue_lock);
470
471 if (drain) {
472 spin_lock_irq(q->queue_lock);
473 __blk_drain_queue(q, false);
474 spin_unlock_irq(q->queue_lock);
475
476
477 synchronize_rcu();
478 }
479}
480EXPORT_SYMBOL_GPL(blk_queue_bypass_start);
481
482
483
484
485
486
487
488void blk_queue_bypass_end(struct request_queue *q)
489{
490 spin_lock_irq(q->queue_lock);
491 if (!--q->bypass_depth)
492 queue_flag_clear(QUEUE_FLAG_BYPASS, q);
493 WARN_ON_ONCE(q->bypass_depth < 0);
494 spin_unlock_irq(q->queue_lock);
495}
496EXPORT_SYMBOL_GPL(blk_queue_bypass_end);
497
498
499
500
501
502
503
504
505void blk_cleanup_queue(struct request_queue *q)
506{
507 spinlock_t *lock = q->queue_lock;
508
509
510 mutex_lock(&q->sysfs_lock);
511 queue_flag_set_unlocked(QUEUE_FLAG_DYING, q);
512 spin_lock_irq(lock);
513
514
515
516
517
518
519
520
521
522
523 q->bypass_depth++;
524 queue_flag_set(QUEUE_FLAG_BYPASS, q);
525
526 queue_flag_set(QUEUE_FLAG_NOMERGES, q);
527 queue_flag_set(QUEUE_FLAG_NOXMERGES, q);
528 queue_flag_set(QUEUE_FLAG_DYING, q);
529 spin_unlock_irq(lock);
530 mutex_unlock(&q->sysfs_lock);
531
532
533
534
535
536 spin_lock_irq(lock);
537 __blk_drain_queue(q, true);
538 queue_flag_set(QUEUE_FLAG_DEAD, q);
539 spin_unlock_irq(lock);
540
541
542 del_timer_sync(&q->backing_dev_info.laptop_mode_wb_timer);
543 blk_sync_queue(q);
544
545 spin_lock_irq(lock);
546 if (q->queue_lock != &q->__queue_lock)
547 q->queue_lock = &q->__queue_lock;
548 spin_unlock_irq(lock);
549
550
551 blk_put_queue(q);
552}
553EXPORT_SYMBOL(blk_cleanup_queue);
554
555int blk_init_rl(struct request_list *rl, struct request_queue *q,
556 gfp_t gfp_mask)
557{
558 if (unlikely(rl->rq_pool))
559 return 0;
560
561 rl->q = q;
562 rl->count[BLK_RW_SYNC] = rl->count[BLK_RW_ASYNC] = 0;
563 rl->starved[BLK_RW_SYNC] = rl->starved[BLK_RW_ASYNC] = 0;
564 init_waitqueue_head(&rl->wait[BLK_RW_SYNC]);
565 init_waitqueue_head(&rl->wait[BLK_RW_ASYNC]);
566
567 rl->rq_pool = mempool_create_node(BLKDEV_MIN_RQ, mempool_alloc_slab,
568 mempool_free_slab, request_cachep,
569 gfp_mask, q->node);
570 if (!rl->rq_pool)
571 return -ENOMEM;
572
573 return 0;
574}
575
576void blk_exit_rl(struct request_list *rl)
577{
578 if (rl->rq_pool)
579 mempool_destroy(rl->rq_pool);
580}
581
582struct request_queue *blk_alloc_queue(gfp_t gfp_mask)
583{
584 return blk_alloc_queue_node(gfp_mask, NUMA_NO_NODE);
585}
586EXPORT_SYMBOL(blk_alloc_queue);
587
588struct request_queue *blk_alloc_queue_node(gfp_t gfp_mask, int node_id)
589{
590 struct request_queue *q;
591 int err;
592
593 q = kmem_cache_alloc_node(blk_requestq_cachep,
594 gfp_mask | __GFP_ZERO, node_id);
595 if (!q)
596 return NULL;
597
598 q->id = ida_simple_get(&blk_queue_ida, 0, 0, gfp_mask);
599 if (q->id < 0)
600 goto fail_q;
601
602 q->backing_dev_info.ra_pages =
603 (VM_MAX_READAHEAD * 1024) / PAGE_CACHE_SIZE;
604 q->backing_dev_info.state = 0;
605 q->backing_dev_info.capabilities = BDI_CAP_MAP_COPY;
606 q->backing_dev_info.name = "block";
607 q->node = node_id;
608
609 err = bdi_init(&q->backing_dev_info);
610 if (err)
611 goto fail_id;
612
613 setup_timer(&q->backing_dev_info.laptop_mode_wb_timer,
614 laptop_mode_timer_fn, (unsigned long) q);
615 setup_timer(&q->timeout, blk_rq_timed_out_timer, (unsigned long) q);
616 INIT_LIST_HEAD(&q->queue_head);
617 INIT_LIST_HEAD(&q->timeout_list);
618 INIT_LIST_HEAD(&q->icq_list);
619#ifdef CONFIG_BLK_CGROUP
620 INIT_LIST_HEAD(&q->blkg_list);
621#endif
622 INIT_LIST_HEAD(&q->flush_queue[0]);
623 INIT_LIST_HEAD(&q->flush_queue[1]);
624 INIT_LIST_HEAD(&q->flush_data_in_flight);
625 INIT_DELAYED_WORK(&q->delay_work, blk_delay_work);
626
627 kobject_init(&q->kobj, &blk_queue_ktype);
628
629 mutex_init(&q->sysfs_lock);
630 spin_lock_init(&q->__queue_lock);
631
632
633
634
635
636 q->queue_lock = &q->__queue_lock;
637
638
639
640
641
642
643
644 q->bypass_depth = 1;
645 __set_bit(QUEUE_FLAG_BYPASS, &q->queue_flags);
646
647 if (blkcg_init_queue(q))
648 goto fail_id;
649
650 return q;
651
652fail_id:
653 ida_simple_remove(&blk_queue_ida, q->id);
654fail_q:
655 kmem_cache_free(blk_requestq_cachep, q);
656 return NULL;
657}
658EXPORT_SYMBOL(blk_alloc_queue_node);
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693struct request_queue *blk_init_queue(request_fn_proc *rfn, spinlock_t *lock)
694{
695 return blk_init_queue_node(rfn, lock, NUMA_NO_NODE);
696}
697EXPORT_SYMBOL(blk_init_queue);
698
699struct request_queue *
700blk_init_queue_node(request_fn_proc *rfn, spinlock_t *lock, int node_id)
701{
702 struct request_queue *uninit_q, *q;
703
704 uninit_q = blk_alloc_queue_node(GFP_KERNEL, node_id);
705 if (!uninit_q)
706 return NULL;
707
708 q = blk_init_allocated_queue(uninit_q, rfn, lock);
709 if (!q)
710 blk_cleanup_queue(uninit_q);
711
712 return q;
713}
714EXPORT_SYMBOL(blk_init_queue_node);
715
716struct request_queue *
717blk_init_allocated_queue(struct request_queue *q, request_fn_proc *rfn,
718 spinlock_t *lock)
719{
720 if (!q)
721 return NULL;
722
723 if (blk_init_rl(&q->root_rl, q, GFP_KERNEL))
724 return NULL;
725
726 q->request_fn = rfn;
727 q->prep_rq_fn = NULL;
728 q->unprep_rq_fn = NULL;
729 q->queue_flags |= QUEUE_FLAG_DEFAULT;
730
731
732 if (lock)
733 q->queue_lock = lock;
734
735
736
737
738 blk_queue_make_request(q, blk_queue_bio);
739
740 q->sg_reserved_size = INT_MAX;
741
742
743 if (elevator_init(q, NULL))
744 return NULL;
745 return q;
746}
747EXPORT_SYMBOL(blk_init_allocated_queue);
748
749bool blk_get_queue(struct request_queue *q)
750{
751 if (likely(!blk_queue_dying(q))) {
752 __blk_get_queue(q);
753 return true;
754 }
755
756 return false;
757}
758EXPORT_SYMBOL(blk_get_queue);
759
760static inline void blk_free_request(struct request_list *rl, struct request *rq)
761{
762 if (rq->cmd_flags & REQ_ELVPRIV) {
763 elv_put_request(rl->q, rq);
764 if (rq->elv.icq)
765 put_io_context(rq->elv.icq->ioc);
766 }
767
768 mempool_free(rq, rl->rq_pool);
769}
770
771
772
773
774
775static inline int ioc_batching(struct request_queue *q, struct io_context *ioc)
776{
777 if (!ioc)
778 return 0;
779
780
781
782
783
784
785 return ioc->nr_batch_requests == q->nr_batching ||
786 (ioc->nr_batch_requests > 0
787 && time_before(jiffies, ioc->last_waited + BLK_BATCH_TIME));
788}
789
790
791
792
793
794
795
796static void ioc_set_batching(struct request_queue *q, struct io_context *ioc)
797{
798 if (!ioc || ioc_batching(q, ioc))
799 return;
800
801 ioc->nr_batch_requests = q->nr_batching;
802 ioc->last_waited = jiffies;
803}
804
805static void __freed_request(struct request_list *rl, int sync)
806{
807 struct request_queue *q = rl->q;
808
809
810
811
812
813 if (rl == &q->root_rl &&
814 rl->count[sync] < queue_congestion_off_threshold(q))
815 blk_clear_queue_congested(q, sync);
816
817 if (rl->count[sync] + 1 <= q->nr_requests) {
818 if (waitqueue_active(&rl->wait[sync]))
819 wake_up(&rl->wait[sync]);
820
821 blk_clear_rl_full(rl, sync);
822 }
823}
824
825
826
827
828
829static void freed_request(struct request_list *rl, unsigned int flags)
830{
831 struct request_queue *q = rl->q;
832 int sync = rw_is_sync(flags);
833
834 q->nr_rqs[sync]--;
835 rl->count[sync]--;
836 if (flags & REQ_ELVPRIV)
837 q->nr_rqs_elvpriv--;
838
839 __freed_request(rl, sync);
840
841 if (unlikely(rl->starved[sync ^ 1]))
842 __freed_request(rl, sync ^ 1);
843}
844
845
846
847
848
849static bool blk_rq_should_init_elevator(struct bio *bio)
850{
851 if (!bio)
852 return true;
853
854
855
856
857
858 if (bio->bi_rw & (REQ_FLUSH | REQ_FUA))
859 return false;
860
861 return true;
862}
863
864
865
866
867
868
869
870
871static struct io_context *rq_ioc(struct bio *bio)
872{
873#ifdef CONFIG_BLK_CGROUP
874 if (bio && bio->bi_ioc)
875 return bio->bi_ioc;
876#endif
877 return current->io_context;
878}
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894static struct request *__get_request(struct request_list *rl, int rw_flags,
895 struct bio *bio, gfp_t gfp_mask)
896{
897 struct request_queue *q = rl->q;
898 struct request *rq;
899 struct elevator_type *et = q->elevator->type;
900 struct io_context *ioc = rq_ioc(bio);
901 struct io_cq *icq = NULL;
902 const bool is_sync = rw_is_sync(rw_flags) != 0;
903 int may_queue;
904
905 if (unlikely(blk_queue_dying(q)))
906 return NULL;
907
908 may_queue = elv_may_queue(q, rw_flags);
909 if (may_queue == ELV_MQUEUE_NO)
910 goto rq_starved;
911
912 if (rl->count[is_sync]+1 >= queue_congestion_on_threshold(q)) {
913 if (rl->count[is_sync]+1 >= q->nr_requests) {
914
915
916
917
918
919
920 if (!blk_rl_full(rl, is_sync)) {
921 ioc_set_batching(q, ioc);
922 blk_set_rl_full(rl, is_sync);
923 } else {
924 if (may_queue != ELV_MQUEUE_MUST
925 && !ioc_batching(q, ioc)) {
926
927
928
929
930
931 return NULL;
932 }
933 }
934 }
935
936
937
938
939 if (rl == &q->root_rl)
940 blk_set_queue_congested(q, is_sync);
941 }
942
943
944
945
946
947
948 if (rl->count[is_sync] >= (3 * q->nr_requests / 2))
949 return NULL;
950
951 q->nr_rqs[is_sync]++;
952 rl->count[is_sync]++;
953 rl->starved[is_sync] = 0;
954
955
956
957
958
959
960
961
962
963
964
965 if (blk_rq_should_init_elevator(bio) && !blk_queue_bypass(q)) {
966 rw_flags |= REQ_ELVPRIV;
967 q->nr_rqs_elvpriv++;
968 if (et->icq_cache && ioc)
969 icq = ioc_lookup_icq(ioc, q);
970 }
971
972 if (blk_queue_io_stat(q))
973 rw_flags |= REQ_IO_STAT;
974 spin_unlock_irq(q->queue_lock);
975
976
977 rq = mempool_alloc(rl->rq_pool, gfp_mask);
978 if (!rq)
979 goto fail_alloc;
980
981 blk_rq_init(q, rq);
982 blk_rq_set_rl(rq, rl);
983 rq->cmd_flags = rw_flags | REQ_ALLOCED;
984
985
986 if (rw_flags & REQ_ELVPRIV) {
987 if (unlikely(et->icq_cache && !icq)) {
988 if (ioc)
989 icq = ioc_create_icq(ioc, q, gfp_mask);
990 if (!icq)
991 goto fail_elvpriv;
992 }
993
994 rq->elv.icq = icq;
995 if (unlikely(elv_set_request(q, rq, bio, gfp_mask)))
996 goto fail_elvpriv;
997
998
999 if (icq)
1000 get_io_context(icq->ioc);
1001 }
1002out:
1003
1004
1005
1006
1007
1008
1009 if (ioc_batching(q, ioc))
1010 ioc->nr_batch_requests--;
1011
1012 trace_block_getrq(q, bio, rw_flags & 1);
1013 return rq;
1014
1015fail_elvpriv:
1016
1017
1018
1019
1020
1021
1022 printk_ratelimited(KERN_WARNING "%s: request aux data allocation failed, iosched may be disturbed\n",
1023 dev_name(q->backing_dev_info.dev));
1024
1025 rq->cmd_flags &= ~REQ_ELVPRIV;
1026 rq->elv.icq = NULL;
1027
1028 spin_lock_irq(q->queue_lock);
1029 q->nr_rqs_elvpriv--;
1030 spin_unlock_irq(q->queue_lock);
1031 goto out;
1032
1033fail_alloc:
1034
1035
1036
1037
1038
1039
1040
1041 spin_lock_irq(q->queue_lock);
1042 freed_request(rl, rw_flags);
1043
1044
1045
1046
1047
1048
1049
1050
1051rq_starved:
1052 if (unlikely(rl->count[is_sync] == 0))
1053 rl->starved[is_sync] = 1;
1054 return NULL;
1055}
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071static struct request *get_request(struct request_queue *q, int rw_flags,
1072 struct bio *bio, gfp_t gfp_mask)
1073{
1074 const bool is_sync = rw_is_sync(rw_flags) != 0;
1075 DEFINE_WAIT(wait);
1076 struct request_list *rl;
1077 struct request *rq;
1078
1079 rl = blk_get_rl(q, bio);
1080retry:
1081 rq = __get_request(rl, rw_flags, bio, gfp_mask);
1082 if (rq)
1083 return rq;
1084
1085 if (!(gfp_mask & __GFP_WAIT) || unlikely(blk_queue_dying(q))) {
1086 blk_put_rl(rl);
1087 return NULL;
1088 }
1089
1090
1091 prepare_to_wait_exclusive(&rl->wait[is_sync], &wait,
1092 TASK_UNINTERRUPTIBLE);
1093
1094 trace_block_sleeprq(q, bio, rw_flags & 1);
1095
1096 spin_unlock_irq(q->queue_lock);
1097 io_schedule();
1098
1099
1100
1101
1102
1103
1104 ioc_set_batching(q, current->io_context);
1105
1106 spin_lock_irq(q->queue_lock);
1107 finish_wait(&rl->wait[is_sync], &wait);
1108
1109 goto retry;
1110}
1111
1112struct request *blk_get_request(struct request_queue *q, int rw, gfp_t gfp_mask)
1113{
1114 struct request *rq;
1115
1116 BUG_ON(rw != READ && rw != WRITE);
1117
1118
1119 create_io_context(gfp_mask, q->node);
1120
1121 spin_lock_irq(q->queue_lock);
1122 rq = get_request(q, rw, NULL, gfp_mask);
1123 if (!rq)
1124 spin_unlock_irq(q->queue_lock);
1125
1126
1127 return rq;
1128}
1129EXPORT_SYMBOL(blk_get_request);
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162struct request *blk_make_request(struct request_queue *q, struct bio *bio,
1163 gfp_t gfp_mask)
1164{
1165 struct request *rq = blk_get_request(q, bio_data_dir(bio), gfp_mask);
1166
1167 if (unlikely(!rq))
1168 return ERR_PTR(-ENOMEM);
1169
1170 for_each_bio(bio) {
1171 struct bio *bounce_bio = bio;
1172 int ret;
1173
1174 blk_queue_bounce(q, &bounce_bio);
1175 ret = blk_rq_append_bio(q, rq, bounce_bio);
1176 if (unlikely(ret)) {
1177 blk_put_request(rq);
1178 return ERR_PTR(ret);
1179 }
1180 }
1181
1182 return rq;
1183}
1184EXPORT_SYMBOL(blk_make_request);
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196void blk_requeue_request(struct request_queue *q, struct request *rq)
1197{
1198 blk_delete_timer(rq);
1199 blk_clear_rq_complete(rq);
1200 trace_block_rq_requeue(q, rq);
1201
1202 if (blk_rq_tagged(rq))
1203 blk_queue_end_tag(q, rq);
1204
1205 BUG_ON(blk_queued_rq(rq));
1206
1207 elv_requeue_request(q, rq);
1208}
1209EXPORT_SYMBOL(blk_requeue_request);
1210
1211static void add_acct_request(struct request_queue *q, struct request *rq,
1212 int where)
1213{
1214 drive_stat_acct(rq, 1);
1215 __elv_add_request(q, rq, where);
1216}
1217
1218static void part_round_stats_single(int cpu, struct hd_struct *part,
1219 unsigned long now)
1220{
1221 if (now == part->stamp)
1222 return;
1223
1224 if (part_in_flight(part)) {
1225 __part_stat_add(cpu, part, time_in_queue,
1226 part_in_flight(part) * (now - part->stamp));
1227 __part_stat_add(cpu, part, io_ticks, (now - part->stamp));
1228 }
1229 part->stamp = now;
1230}
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248void part_round_stats(int cpu, struct hd_struct *part)
1249{
1250 unsigned long now = jiffies;
1251
1252 if (part->partno)
1253 part_round_stats_single(cpu, &part_to_disk(part)->part0, now);
1254 part_round_stats_single(cpu, part, now);
1255}
1256EXPORT_SYMBOL_GPL(part_round_stats);
1257
1258#ifdef CONFIG_PM_RUNTIME
1259static void blk_pm_put_request(struct request *rq)
1260{
1261 if (rq->q->dev && !(rq->cmd_flags & REQ_PM) && !--rq->q->nr_pending)
1262 pm_runtime_mark_last_busy(rq->q->dev);
1263}
1264#else
1265static inline void blk_pm_put_request(struct request *rq) {}
1266#endif
1267
1268
1269
1270
1271void __blk_put_request(struct request_queue *q, struct request *req)
1272{
1273 if (unlikely(!q))
1274 return;
1275 if (unlikely(--req->ref_count))
1276 return;
1277
1278 blk_pm_put_request(req);
1279
1280 elv_completed_request(q, req);
1281
1282
1283 WARN_ON(req->bio != NULL);
1284
1285
1286
1287
1288
1289 if (req->cmd_flags & REQ_ALLOCED) {
1290 unsigned int flags = req->cmd_flags;
1291 struct request_list *rl = blk_rq_rl(req);
1292
1293 BUG_ON(!list_empty(&req->queuelist));
1294 BUG_ON(!hlist_unhashed(&req->hash));
1295
1296 blk_free_request(rl, req);
1297 freed_request(rl, flags);
1298 blk_put_rl(rl);
1299 }
1300}
1301EXPORT_SYMBOL_GPL(__blk_put_request);
1302
1303void blk_put_request(struct request *req)
1304{
1305 unsigned long flags;
1306 struct request_queue *q = req->q;
1307
1308 spin_lock_irqsave(q->queue_lock, flags);
1309 __blk_put_request(q, req);
1310 spin_unlock_irqrestore(q->queue_lock, flags);
1311}
1312EXPORT_SYMBOL(blk_put_request);
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327void blk_add_request_payload(struct request *rq, struct page *page,
1328 unsigned int len)
1329{
1330 struct bio *bio = rq->bio;
1331
1332 bio->bi_io_vec->bv_page = page;
1333 bio->bi_io_vec->bv_offset = 0;
1334 bio->bi_io_vec->bv_len = len;
1335
1336 bio->bi_size = len;
1337 bio->bi_vcnt = 1;
1338 bio->bi_phys_segments = 1;
1339
1340 rq->__data_len = rq->resid_len = len;
1341 rq->nr_phys_segments = 1;
1342 rq->buffer = bio_data(bio);
1343}
1344EXPORT_SYMBOL_GPL(blk_add_request_payload);
1345
1346static bool bio_attempt_back_merge(struct request_queue *q, struct request *req,
1347 struct bio *bio)
1348{
1349 const int ff = bio->bi_rw & REQ_FAILFAST_MASK;
1350
1351 if (!ll_back_merge_fn(q, req, bio))
1352 return false;
1353
1354 trace_block_bio_backmerge(q, req, bio);
1355
1356 if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
1357 blk_rq_set_mixed_merge(req);
1358
1359 req->biotail->bi_next = bio;
1360 req->biotail = bio;
1361 req->__data_len += bio->bi_size;
1362 req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));
1363
1364 drive_stat_acct(req, 0);
1365 return true;
1366}
1367
1368static bool bio_attempt_front_merge(struct request_queue *q,
1369 struct request *req, struct bio *bio)
1370{
1371 const int ff = bio->bi_rw & REQ_FAILFAST_MASK;
1372
1373 if (!ll_front_merge_fn(q, req, bio))
1374 return false;
1375
1376 trace_block_bio_frontmerge(q, req, bio);
1377
1378 if ((req->cmd_flags & REQ_FAILFAST_MASK) != ff)
1379 blk_rq_set_mixed_merge(req);
1380
1381 bio->bi_next = req->bio;
1382 req->bio = bio;
1383
1384
1385
1386
1387
1388
1389 req->buffer = bio_data(bio);
1390 req->__sector = bio->bi_sector;
1391 req->__data_len += bio->bi_size;
1392 req->ioprio = ioprio_best(req->ioprio, bio_prio(bio));
1393
1394 drive_stat_acct(req, 0);
1395 return true;
1396}
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415static bool attempt_plug_merge(struct request_queue *q, struct bio *bio,
1416 unsigned int *request_count)
1417{
1418 struct blk_plug *plug;
1419 struct request *rq;
1420 bool ret = false;
1421
1422 plug = current->plug;
1423 if (!plug)
1424 goto out;
1425 *request_count = 0;
1426
1427 list_for_each_entry_reverse(rq, &plug->list, queuelist) {
1428 int el_ret;
1429
1430 if (rq->q == q)
1431 (*request_count)++;
1432
1433 if (rq->q != q || !blk_rq_merge_ok(rq, bio))
1434 continue;
1435
1436 el_ret = blk_try_merge(rq, bio);
1437 if (el_ret == ELEVATOR_BACK_MERGE) {
1438 ret = bio_attempt_back_merge(q, rq, bio);
1439 if (ret)
1440 break;
1441 } else if (el_ret == ELEVATOR_FRONT_MERGE) {
1442 ret = bio_attempt_front_merge(q, rq, bio);
1443 if (ret)
1444 break;
1445 }
1446 }
1447out:
1448 return ret;
1449}
1450
1451void init_request_from_bio(struct request *req, struct bio *bio)
1452{
1453 req->cmd_type = REQ_TYPE_FS;
1454
1455 req->cmd_flags |= bio->bi_rw & REQ_COMMON_MASK;
1456 if (bio->bi_rw & REQ_RAHEAD)
1457 req->cmd_flags |= REQ_FAILFAST_MASK;
1458
1459 req->errors = 0;
1460 req->__sector = bio->bi_sector;
1461 req->ioprio = bio_prio(bio);
1462 blk_rq_bio_prep(req->q, req, bio);
1463}
1464
1465void blk_queue_bio(struct request_queue *q, struct bio *bio)
1466{
1467 const bool sync = !!(bio->bi_rw & REQ_SYNC);
1468 struct blk_plug *plug;
1469 int el_ret, rw_flags, where = ELEVATOR_INSERT_SORT;
1470 struct request *req;
1471 unsigned int request_count = 0;
1472
1473
1474
1475
1476
1477
1478 blk_queue_bounce(q, &bio);
1479
1480 if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
1481 bio_endio(bio, -EIO);
1482 return;
1483 }
1484
1485 if (bio->bi_rw & (REQ_FLUSH | REQ_FUA)) {
1486 spin_lock_irq(q->queue_lock);
1487 where = ELEVATOR_INSERT_FLUSH;
1488 goto get_rq;
1489 }
1490
1491
1492
1493
1494
1495 if (attempt_plug_merge(q, bio, &request_count))
1496 return;
1497
1498 spin_lock_irq(q->queue_lock);
1499
1500 el_ret = elv_merge(q, &req, bio);
1501 if (el_ret == ELEVATOR_BACK_MERGE) {
1502 if (bio_attempt_back_merge(q, req, bio)) {
1503 elv_bio_merged(q, req, bio);
1504 if (!attempt_back_merge(q, req))
1505 elv_merged_request(q, req, el_ret);
1506 goto out_unlock;
1507 }
1508 } else if (el_ret == ELEVATOR_FRONT_MERGE) {
1509 if (bio_attempt_front_merge(q, req, bio)) {
1510 elv_bio_merged(q, req, bio);
1511 if (!attempt_front_merge(q, req))
1512 elv_merged_request(q, req, el_ret);
1513 goto out_unlock;
1514 }
1515 }
1516
1517get_rq:
1518
1519
1520
1521
1522
1523 rw_flags = bio_data_dir(bio);
1524 if (sync)
1525 rw_flags |= REQ_SYNC;
1526
1527
1528
1529
1530
1531 req = get_request(q, rw_flags, bio, GFP_NOIO);
1532 if (unlikely(!req)) {
1533 bio_endio(bio, -ENODEV);
1534 goto out_unlock;
1535 }
1536
1537
1538
1539
1540
1541
1542
1543 init_request_from_bio(req, bio);
1544
1545 if (test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags))
1546 req->cpu = raw_smp_processor_id();
1547
1548 plug = current->plug;
1549 if (plug) {
1550
1551
1552
1553
1554
1555
1556 if (list_empty(&plug->list))
1557 trace_block_plug(q);
1558 else {
1559 if (request_count >= BLK_MAX_REQUEST_COUNT) {
1560 blk_flush_plug_list(plug, false);
1561 trace_block_plug(q);
1562 }
1563 }
1564 list_add_tail(&req->queuelist, &plug->list);
1565 drive_stat_acct(req, 1);
1566 } else {
1567 spin_lock_irq(q->queue_lock);
1568 add_acct_request(q, req, where);
1569 __blk_run_queue(q);
1570out_unlock:
1571 spin_unlock_irq(q->queue_lock);
1572 }
1573}
1574EXPORT_SYMBOL_GPL(blk_queue_bio);
1575
1576
1577
1578
1579static inline void blk_partition_remap(struct bio *bio)
1580{
1581 struct block_device *bdev = bio->bi_bdev;
1582
1583 if (bio_sectors(bio) && bdev != bdev->bd_contains) {
1584 struct hd_struct *p = bdev->bd_part;
1585
1586 bio->bi_sector += p->start_sect;
1587 bio->bi_bdev = bdev->bd_contains;
1588
1589 trace_block_bio_remap(bdev_get_queue(bio->bi_bdev), bio,
1590 bdev->bd_dev,
1591 bio->bi_sector - p->start_sect);
1592 }
1593}
1594
1595static void handle_bad_sector(struct bio *bio)
1596{
1597 char b[BDEVNAME_SIZE];
1598
1599 printk(KERN_INFO "attempt to access beyond end of device\n");
1600 printk(KERN_INFO "%s: rw=%ld, want=%Lu, limit=%Lu\n",
1601 bdevname(bio->bi_bdev, b),
1602 bio->bi_rw,
1603 (unsigned long long)bio_end_sector(bio),
1604 (long long)(i_size_read(bio->bi_bdev->bd_inode) >> 9));
1605
1606 set_bit(BIO_EOF, &bio->bi_flags);
1607}
1608
1609#ifdef CONFIG_FAIL_MAKE_REQUEST
1610
1611static DECLARE_FAULT_ATTR(fail_make_request);
1612
1613static int __init setup_fail_make_request(char *str)
1614{
1615 return setup_fault_attr(&fail_make_request, str);
1616}
1617__setup("fail_make_request=", setup_fail_make_request);
1618
1619static bool should_fail_request(struct hd_struct *part, unsigned int bytes)
1620{
1621 return part->make_it_fail && should_fail(&fail_make_request, bytes);
1622}
1623
1624static int __init fail_make_request_debugfs(void)
1625{
1626 struct dentry *dir = fault_create_debugfs_attr("fail_make_request",
1627 NULL, &fail_make_request);
1628
1629 return IS_ERR(dir) ? PTR_ERR(dir) : 0;
1630}
1631
1632late_initcall(fail_make_request_debugfs);
1633
1634#else
1635
1636static inline bool should_fail_request(struct hd_struct *part,
1637 unsigned int bytes)
1638{
1639 return false;
1640}
1641
1642#endif
1643
1644
1645
1646
1647static inline int bio_check_eod(struct bio *bio, unsigned int nr_sectors)
1648{
1649 sector_t maxsector;
1650
1651 if (!nr_sectors)
1652 return 0;
1653
1654
1655 maxsector = i_size_read(bio->bi_bdev->bd_inode) >> 9;
1656 if (maxsector) {
1657 sector_t sector = bio->bi_sector;
1658
1659 if (maxsector < nr_sectors || maxsector - nr_sectors < sector) {
1660
1661
1662
1663
1664
1665 handle_bad_sector(bio);
1666 return 1;
1667 }
1668 }
1669
1670 return 0;
1671}
1672
1673static noinline_for_stack bool
1674generic_make_request_checks(struct bio *bio)
1675{
1676 struct request_queue *q;
1677 int nr_sectors = bio_sectors(bio);
1678 int err = -EIO;
1679 char b[BDEVNAME_SIZE];
1680 struct hd_struct *part;
1681
1682 might_sleep();
1683
1684 if (bio_check_eod(bio, nr_sectors))
1685 goto end_io;
1686
1687 q = bdev_get_queue(bio->bi_bdev);
1688 if (unlikely(!q)) {
1689 printk(KERN_ERR
1690 "generic_make_request: Trying to access "
1691 "nonexistent block-device %s (%Lu)\n",
1692 bdevname(bio->bi_bdev, b),
1693 (long long) bio->bi_sector);
1694 goto end_io;
1695 }
1696
1697 if (likely(bio_is_rw(bio) &&
1698 nr_sectors > queue_max_hw_sectors(q))) {
1699 printk(KERN_ERR "bio too big device %s (%u > %u)\n",
1700 bdevname(bio->bi_bdev, b),
1701 bio_sectors(bio),
1702 queue_max_hw_sectors(q));
1703 goto end_io;
1704 }
1705
1706 part = bio->bi_bdev->bd_part;
1707 if (should_fail_request(part, bio->bi_size) ||
1708 should_fail_request(&part_to_disk(part)->part0,
1709 bio->bi_size))
1710 goto end_io;
1711
1712
1713
1714
1715
1716 blk_partition_remap(bio);
1717
1718 if (bio_check_eod(bio, nr_sectors))
1719 goto end_io;
1720
1721
1722
1723
1724
1725
1726 if ((bio->bi_rw & (REQ_FLUSH | REQ_FUA)) && !q->flush_flags) {
1727 bio->bi_rw &= ~(REQ_FLUSH | REQ_FUA);
1728 if (!nr_sectors) {
1729 err = 0;
1730 goto end_io;
1731 }
1732 }
1733
1734 if ((bio->bi_rw & REQ_DISCARD) &&
1735 (!blk_queue_discard(q) ||
1736 ((bio->bi_rw & REQ_SECURE) && !blk_queue_secdiscard(q)))) {
1737 err = -EOPNOTSUPP;
1738 goto end_io;
1739 }
1740
1741 if (bio->bi_rw & REQ_WRITE_SAME && !bdev_write_same(bio->bi_bdev)) {
1742 err = -EOPNOTSUPP;
1743 goto end_io;
1744 }
1745
1746
1747
1748
1749
1750
1751
1752 create_io_context(GFP_ATOMIC, q->node);
1753
1754 if (blk_throtl_bio(q, bio))
1755 return false;
1756
1757 trace_block_bio_queue(q, bio);
1758 return true;
1759
1760end_io:
1761 bio_endio(bio, err);
1762 return false;
1763}
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789void generic_make_request(struct bio *bio)
1790{
1791 struct bio_list bio_list_on_stack;
1792
1793 if (!generic_make_request_checks(bio))
1794 return;
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806 if (current->bio_list) {
1807 bio_list_add(current->bio_list, bio);
1808 return;
1809 }
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825 BUG_ON(bio->bi_next);
1826 bio_list_init(&bio_list_on_stack);
1827 current->bio_list = &bio_list_on_stack;
1828 do {
1829 struct request_queue *q = bdev_get_queue(bio->bi_bdev);
1830
1831 q->make_request_fn(q, bio);
1832
1833 bio = bio_list_pop(current->bio_list);
1834 } while (bio);
1835 current->bio_list = NULL;
1836}
1837EXPORT_SYMBOL(generic_make_request);
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849void submit_bio(int rw, struct bio *bio)
1850{
1851 bio->bi_rw |= rw;
1852
1853
1854
1855
1856
1857 if (bio_has_data(bio)) {
1858 unsigned int count;
1859
1860 if (unlikely(rw & REQ_WRITE_SAME))
1861 count = bdev_logical_block_size(bio->bi_bdev) >> 9;
1862 else
1863 count = bio_sectors(bio);
1864
1865 if (rw & WRITE) {
1866 count_vm_events(PGPGOUT, count);
1867 } else {
1868 task_io_account_read(bio->bi_size);
1869 count_vm_events(PGPGIN, count);
1870 }
1871
1872 if (unlikely(block_dump)) {
1873 char b[BDEVNAME_SIZE];
1874 printk(KERN_DEBUG "%s(%d): %s block %Lu on %s (%u sectors)\n",
1875 current->comm, task_pid_nr(current),
1876 (rw & WRITE) ? "WRITE" : "READ",
1877 (unsigned long long)bio->bi_sector,
1878 bdevname(bio->bi_bdev, b),
1879 count);
1880 }
1881 }
1882
1883 generic_make_request(bio);
1884}
1885EXPORT_SYMBOL(submit_bio);
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908int blk_rq_check_limits(struct request_queue *q, struct request *rq)
1909{
1910 if (!rq_mergeable(rq))
1911 return 0;
1912
1913 if (blk_rq_sectors(rq) > blk_queue_get_max_sectors(q, rq->cmd_flags)) {
1914 printk(KERN_ERR "%s: over max size limit.\n", __func__);
1915 return -EIO;
1916 }
1917
1918
1919
1920
1921
1922
1923
1924 blk_recalc_rq_segments(rq);
1925 if (rq->nr_phys_segments > queue_max_segments(q)) {
1926 printk(KERN_ERR "%s: over max segments limit.\n", __func__);
1927 return -EIO;
1928 }
1929
1930 return 0;
1931}
1932EXPORT_SYMBOL_GPL(blk_rq_check_limits);
1933
1934
1935
1936
1937
1938
1939int blk_insert_cloned_request(struct request_queue *q, struct request *rq)
1940{
1941 unsigned long flags;
1942 int where = ELEVATOR_INSERT_BACK;
1943
1944 if (blk_rq_check_limits(q, rq))
1945 return -EIO;
1946
1947 if (rq->rq_disk &&
1948 should_fail_request(&rq->rq_disk->part0, blk_rq_bytes(rq)))
1949 return -EIO;
1950
1951 spin_lock_irqsave(q->queue_lock, flags);
1952 if (unlikely(blk_queue_dying(q))) {
1953 spin_unlock_irqrestore(q->queue_lock, flags);
1954 return -ENODEV;
1955 }
1956
1957
1958
1959
1960
1961 BUG_ON(blk_queued_rq(rq));
1962
1963 if (rq->cmd_flags & (REQ_FLUSH|REQ_FUA))
1964 where = ELEVATOR_INSERT_FLUSH;
1965
1966 add_acct_request(q, rq, where);
1967 if (where == ELEVATOR_INSERT_FLUSH)
1968 __blk_run_queue(q);
1969 spin_unlock_irqrestore(q->queue_lock, flags);
1970
1971 return 0;
1972}
1973EXPORT_SYMBOL_GPL(blk_insert_cloned_request);
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991unsigned int blk_rq_err_bytes(const struct request *rq)
1992{
1993 unsigned int ff = rq->cmd_flags & REQ_FAILFAST_MASK;
1994 unsigned int bytes = 0;
1995 struct bio *bio;
1996
1997 if (!(rq->cmd_flags & REQ_MIXED_MERGE))
1998 return blk_rq_bytes(rq);
1999
2000
2001
2002
2003
2004
2005
2006
2007 for (bio = rq->bio; bio; bio = bio->bi_next) {
2008 if ((bio->bi_rw & ff) != ff)
2009 break;
2010 bytes += bio->bi_size;
2011 }
2012
2013
2014 BUG_ON(blk_rq_bytes(rq) && !bytes);
2015 return bytes;
2016}
2017EXPORT_SYMBOL_GPL(blk_rq_err_bytes);
2018
2019static void blk_account_io_completion(struct request *req, unsigned int bytes)
2020{
2021 if (blk_do_io_stat(req)) {
2022 const int rw = rq_data_dir(req);
2023 struct hd_struct *part;
2024 int cpu;
2025
2026 cpu = part_stat_lock();
2027 part = req->part;
2028 part_stat_add(cpu, part, sectors[rw], bytes >> 9);
2029 part_stat_unlock();
2030 }
2031}
2032
2033static void blk_account_io_done(struct request *req)
2034{
2035
2036
2037
2038
2039
2040 if (blk_do_io_stat(req) && !(req->cmd_flags & REQ_FLUSH_SEQ)) {
2041 unsigned long duration = jiffies - req->start_time;
2042 const int rw = rq_data_dir(req);
2043 struct hd_struct *part;
2044 int cpu;
2045
2046 cpu = part_stat_lock();
2047 part = req->part;
2048
2049 part_stat_inc(cpu, part, ios[rw]);
2050 part_stat_add(cpu, part, ticks[rw], duration);
2051 part_round_stats(cpu, part);
2052 part_dec_in_flight(part, rw);
2053
2054 hd_struct_put(part);
2055 part_stat_unlock();
2056 }
2057}
2058
2059#ifdef CONFIG_PM_RUNTIME
2060
2061
2062
2063
2064static struct request *blk_pm_peek_request(struct request_queue *q,
2065 struct request *rq)
2066{
2067 if (q->dev && (q->rpm_status == RPM_SUSPENDED ||
2068 (q->rpm_status != RPM_ACTIVE && !(rq->cmd_flags & REQ_PM))))
2069 return NULL;
2070 else
2071 return rq;
2072}
2073#else
2074static inline struct request *blk_pm_peek_request(struct request_queue *q,
2075 struct request *rq)
2076{
2077 return rq;
2078}
2079#endif
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097struct request *blk_peek_request(struct request_queue *q)
2098{
2099 struct request *rq;
2100 int ret;
2101
2102 while ((rq = __elv_next_request(q)) != NULL) {
2103
2104 rq = blk_pm_peek_request(q, rq);
2105 if (!rq)
2106 break;
2107
2108 if (!(rq->cmd_flags & REQ_STARTED)) {
2109
2110
2111
2112
2113
2114 if (rq->cmd_flags & REQ_SORTED)
2115 elv_activate_rq(q, rq);
2116
2117
2118
2119
2120
2121
2122 rq->cmd_flags |= REQ_STARTED;
2123 trace_block_rq_issue(q, rq);
2124 }
2125
2126 if (!q->boundary_rq || q->boundary_rq == rq) {
2127 q->end_sector = rq_end_sector(rq);
2128 q->boundary_rq = NULL;
2129 }
2130
2131 if (rq->cmd_flags & REQ_DONTPREP)
2132 break;
2133
2134 if (q->dma_drain_size && blk_rq_bytes(rq)) {
2135
2136
2137
2138
2139
2140
2141 rq->nr_phys_segments++;
2142 }
2143
2144 if (!q->prep_rq_fn)
2145 break;
2146
2147 ret = q->prep_rq_fn(q, rq);
2148 if (ret == BLKPREP_OK) {
2149 break;
2150 } else if (ret == BLKPREP_DEFER) {
2151
2152
2153
2154
2155
2156
2157 if (q->dma_drain_size && blk_rq_bytes(rq) &&
2158 !(rq->cmd_flags & REQ_DONTPREP)) {
2159
2160
2161
2162
2163 --rq->nr_phys_segments;
2164 }
2165
2166 rq = NULL;
2167 break;
2168 } else if (ret == BLKPREP_KILL) {
2169 rq->cmd_flags |= REQ_QUIET;
2170
2171
2172
2173
2174 blk_start_request(rq);
2175 __blk_end_request_all(rq, -EIO);
2176 } else {
2177 printk(KERN_ERR "%s: bad return=%d\n", __func__, ret);
2178 break;
2179 }
2180 }
2181
2182 return rq;
2183}
2184EXPORT_SYMBOL(blk_peek_request);
2185
2186void blk_dequeue_request(struct request *rq)
2187{
2188 struct request_queue *q = rq->q;
2189
2190 BUG_ON(list_empty(&rq->queuelist));
2191 BUG_ON(ELV_ON_HASH(rq));
2192
2193 list_del_init(&rq->queuelist);
2194
2195
2196
2197
2198
2199
2200 if (blk_account_rq(rq)) {
2201 q->in_flight[rq_is_sync(rq)]++;
2202 set_io_start_time_ns(rq);
2203 }
2204}
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220void blk_start_request(struct request *req)
2221{
2222 blk_dequeue_request(req);
2223
2224
2225
2226
2227
2228 req->resid_len = blk_rq_bytes(req);
2229 if (unlikely(blk_bidi_rq(req)))
2230 req->next_rq->resid_len = blk_rq_bytes(req->next_rq);
2231
2232 blk_add_timer(req);
2233}
2234EXPORT_SYMBOL(blk_start_request);
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251struct request *blk_fetch_request(struct request_queue *q)
2252{
2253 struct request *rq;
2254
2255 rq = blk_peek_request(q);
2256 if (rq)
2257 blk_start_request(rq);
2258 return rq;
2259}
2260EXPORT_SYMBOL(blk_fetch_request);
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284bool blk_update_request(struct request *req, int error, unsigned int nr_bytes)
2285{
2286 int total_bytes;
2287
2288 if (!req->bio)
2289 return false;
2290
2291 trace_block_rq_complete(req->q, req);
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301 if (req->cmd_type == REQ_TYPE_FS)
2302 req->errors = 0;
2303
2304 if (error && req->cmd_type == REQ_TYPE_FS &&
2305 !(req->cmd_flags & REQ_QUIET)) {
2306 char *error_type;
2307
2308 switch (error) {
2309 case -ENOLINK:
2310 error_type = "recoverable transport";
2311 break;
2312 case -EREMOTEIO:
2313 error_type = "critical target";
2314 break;
2315 case -EBADE:
2316 error_type = "critical nexus";
2317 break;
2318 case -ETIMEDOUT:
2319 error_type = "timeout";
2320 break;
2321 case -EIO:
2322 default:
2323 error_type = "I/O";
2324 break;
2325 }
2326 printk_ratelimited(KERN_ERR "end_request: %s error, dev %s, sector %llu\n",
2327 error_type, req->rq_disk ?
2328 req->rq_disk->disk_name : "?",
2329 (unsigned long long)blk_rq_pos(req));
2330
2331 }
2332
2333 blk_account_io_completion(req, nr_bytes);
2334
2335 total_bytes = 0;
2336 while (req->bio) {
2337 struct bio *bio = req->bio;
2338 unsigned bio_bytes = min(bio->bi_size, nr_bytes);
2339
2340 if (bio_bytes == bio->bi_size)
2341 req->bio = bio->bi_next;
2342
2343 req_bio_endio(req, bio, bio_bytes, error);
2344
2345 total_bytes += bio_bytes;
2346 nr_bytes -= bio_bytes;
2347
2348 if (!nr_bytes)
2349 break;
2350 }
2351
2352
2353
2354
2355 if (!req->bio) {
2356
2357
2358
2359
2360
2361 req->__data_len = 0;
2362 return false;
2363 }
2364
2365 req->__data_len -= total_bytes;
2366 req->buffer = bio_data(req->bio);
2367
2368
2369 if (req->cmd_type == REQ_TYPE_FS)
2370 req->__sector += total_bytes >> 9;
2371
2372
2373 if (req->cmd_flags & REQ_MIXED_MERGE) {
2374 req->cmd_flags &= ~REQ_FAILFAST_MASK;
2375 req->cmd_flags |= req->bio->bi_rw & REQ_FAILFAST_MASK;
2376 }
2377
2378
2379
2380
2381
2382 if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {
2383 blk_dump_rq_flags(req, "request botched");
2384 req->__data_len = blk_rq_cur_bytes(req);
2385 }
2386
2387
2388 blk_recalc_rq_segments(req);
2389
2390 return true;
2391}
2392EXPORT_SYMBOL_GPL(blk_update_request);
2393
2394static bool blk_update_bidi_request(struct request *rq, int error,
2395 unsigned int nr_bytes,
2396 unsigned int bidi_bytes)
2397{
2398 if (blk_update_request(rq, error, nr_bytes))
2399 return true;
2400
2401
2402 if (unlikely(blk_bidi_rq(rq)) &&
2403 blk_update_request(rq->next_rq, error, bidi_bytes))
2404 return true;
2405
2406 if (blk_queue_add_random(rq->q))
2407 add_disk_randomness(rq->rq_disk);
2408
2409 return false;
2410}
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422void blk_unprep_request(struct request *req)
2423{
2424 struct request_queue *q = req->q;
2425
2426 req->cmd_flags &= ~REQ_DONTPREP;
2427 if (q->unprep_rq_fn)
2428 q->unprep_rq_fn(q, req);
2429}
2430EXPORT_SYMBOL_GPL(blk_unprep_request);
2431
2432
2433
2434
2435static void blk_finish_request(struct request *req, int error)
2436{
2437 if (blk_rq_tagged(req))
2438 blk_queue_end_tag(req->q, req);
2439
2440 BUG_ON(blk_queued_rq(req));
2441
2442 if (unlikely(laptop_mode) && req->cmd_type == REQ_TYPE_FS)
2443 laptop_io_completion(&req->q->backing_dev_info);
2444
2445 blk_delete_timer(req);
2446
2447 if (req->cmd_flags & REQ_DONTPREP)
2448 blk_unprep_request(req);
2449
2450
2451 blk_account_io_done(req);
2452
2453 if (req->end_io)
2454 req->end_io(req, error);
2455 else {
2456 if (blk_bidi_rq(req))
2457 __blk_put_request(req->next_rq->q, req->next_rq);
2458
2459 __blk_put_request(req->q, req);
2460 }
2461}
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480static bool blk_end_bidi_request(struct request *rq, int error,
2481 unsigned int nr_bytes, unsigned int bidi_bytes)
2482{
2483 struct request_queue *q = rq->q;
2484 unsigned long flags;
2485
2486 if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes))
2487 return true;
2488
2489 spin_lock_irqsave(q->queue_lock, flags);
2490 blk_finish_request(rq, error);
2491 spin_unlock_irqrestore(q->queue_lock, flags);
2492
2493 return false;
2494}
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511bool __blk_end_bidi_request(struct request *rq, int error,
2512 unsigned int nr_bytes, unsigned int bidi_bytes)
2513{
2514 if (blk_update_bidi_request(rq, error, nr_bytes, bidi_bytes))
2515 return true;
2516
2517 blk_finish_request(rq, error);
2518
2519 return false;
2520}
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536bool blk_end_request(struct request *rq, int error, unsigned int nr_bytes)
2537{
2538 return blk_end_bidi_request(rq, error, nr_bytes, 0);
2539}
2540EXPORT_SYMBOL(blk_end_request);
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550void blk_end_request_all(struct request *rq, int error)
2551{
2552 bool pending;
2553 unsigned int bidi_bytes = 0;
2554
2555 if (unlikely(blk_bidi_rq(rq)))
2556 bidi_bytes = blk_rq_bytes(rq->next_rq);
2557
2558 pending = blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes);
2559 BUG_ON(pending);
2560}
2561EXPORT_SYMBOL(blk_end_request_all);
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575bool blk_end_request_cur(struct request *rq, int error)
2576{
2577 return blk_end_request(rq, error, blk_rq_cur_bytes(rq));
2578}
2579EXPORT_SYMBOL(blk_end_request_cur);
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593bool blk_end_request_err(struct request *rq, int error)
2594{
2595 WARN_ON(error >= 0);
2596 return blk_end_request(rq, error, blk_rq_err_bytes(rq));
2597}
2598EXPORT_SYMBOL_GPL(blk_end_request_err);
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613bool __blk_end_request(struct request *rq, int error, unsigned int nr_bytes)
2614{
2615 return __blk_end_bidi_request(rq, error, nr_bytes, 0);
2616}
2617EXPORT_SYMBOL(__blk_end_request);
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627void __blk_end_request_all(struct request *rq, int error)
2628{
2629 bool pending;
2630 unsigned int bidi_bytes = 0;
2631
2632 if (unlikely(blk_bidi_rq(rq)))
2633 bidi_bytes = blk_rq_bytes(rq->next_rq);
2634
2635 pending = __blk_end_bidi_request(rq, error, blk_rq_bytes(rq), bidi_bytes);
2636 BUG_ON(pending);
2637}
2638EXPORT_SYMBOL(__blk_end_request_all);
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653bool __blk_end_request_cur(struct request *rq, int error)
2654{
2655 return __blk_end_request(rq, error, blk_rq_cur_bytes(rq));
2656}
2657EXPORT_SYMBOL(__blk_end_request_cur);
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672bool __blk_end_request_err(struct request *rq, int error)
2673{
2674 WARN_ON(error >= 0);
2675 return __blk_end_request(rq, error, blk_rq_err_bytes(rq));
2676}
2677EXPORT_SYMBOL_GPL(__blk_end_request_err);
2678
2679void blk_rq_bio_prep(struct request_queue *q, struct request *rq,
2680 struct bio *bio)
2681{
2682
2683 rq->cmd_flags |= bio->bi_rw & REQ_WRITE;
2684
2685 if (bio_has_data(bio)) {
2686 rq->nr_phys_segments = bio_phys_segments(q, bio);
2687 rq->buffer = bio_data(bio);
2688 }
2689 rq->__data_len = bio->bi_size;
2690 rq->bio = rq->biotail = bio;
2691
2692 if (bio->bi_bdev)
2693 rq->rq_disk = bio->bi_bdev->bd_disk;
2694}
2695
2696#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
2697
2698
2699
2700
2701
2702
2703
2704void rq_flush_dcache_pages(struct request *rq)
2705{
2706 struct req_iterator iter;
2707 struct bio_vec *bvec;
2708
2709 rq_for_each_segment(bvec, rq, iter)
2710 flush_dcache_page(bvec->bv_page);
2711}
2712EXPORT_SYMBOL_GPL(rq_flush_dcache_pages);
2713#endif
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734int blk_lld_busy(struct request_queue *q)
2735{
2736 if (q->lld_busy_fn)
2737 return q->lld_busy_fn(q);
2738
2739 return 0;
2740}
2741EXPORT_SYMBOL_GPL(blk_lld_busy);
2742
2743
2744
2745
2746
2747
2748
2749
2750void blk_rq_unprep_clone(struct request *rq)
2751{
2752 struct bio *bio;
2753
2754 while ((bio = rq->bio) != NULL) {
2755 rq->bio = bio->bi_next;
2756
2757 bio_put(bio);
2758 }
2759}
2760EXPORT_SYMBOL_GPL(blk_rq_unprep_clone);
2761
2762
2763
2764
2765
2766static void __blk_rq_prep_clone(struct request *dst, struct request *src)
2767{
2768 dst->cpu = src->cpu;
2769 dst->cmd_flags = (src->cmd_flags & REQ_CLONE_MASK) | REQ_NOMERGE;
2770 dst->cmd_type = src->cmd_type;
2771 dst->__sector = blk_rq_pos(src);
2772 dst->__data_len = blk_rq_bytes(src);
2773 dst->nr_phys_segments = src->nr_phys_segments;
2774 dst->ioprio = src->ioprio;
2775 dst->extra_len = src->extra_len;
2776}
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
2798 struct bio_set *bs, gfp_t gfp_mask,
2799 int (*bio_ctr)(struct bio *, struct bio *, void *),
2800 void *data)
2801{
2802 struct bio *bio, *bio_src;
2803
2804 if (!bs)
2805 bs = fs_bio_set;
2806
2807 blk_rq_init(NULL, rq);
2808
2809 __rq_for_each_bio(bio_src, rq_src) {
2810 bio = bio_clone_bioset(bio_src, gfp_mask, bs);
2811 if (!bio)
2812 goto free_and_out;
2813
2814 if (bio_ctr && bio_ctr(bio, bio_src, data))
2815 goto free_and_out;
2816
2817 if (rq->bio) {
2818 rq->biotail->bi_next = bio;
2819 rq->biotail = bio;
2820 } else
2821 rq->bio = rq->biotail = bio;
2822 }
2823
2824 __blk_rq_prep_clone(rq, rq_src);
2825
2826 return 0;
2827
2828free_and_out:
2829 if (bio)
2830 bio_put(bio);
2831 blk_rq_unprep_clone(rq);
2832
2833 return -ENOMEM;
2834}
2835EXPORT_SYMBOL_GPL(blk_rq_prep_clone);
2836
2837int kblockd_schedule_work(struct request_queue *q, struct work_struct *work)
2838{
2839 return queue_work(kblockd_workqueue, work);
2840}
2841EXPORT_SYMBOL(kblockd_schedule_work);
2842
2843int kblockd_schedule_delayed_work(struct request_queue *q,
2844 struct delayed_work *dwork, unsigned long delay)
2845{
2846 return queue_delayed_work(kblockd_workqueue, dwork, delay);
2847}
2848EXPORT_SYMBOL(kblockd_schedule_delayed_work);
2849
2850#define PLUG_MAGIC 0x91827364
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866void blk_start_plug(struct blk_plug *plug)
2867{
2868 struct task_struct *tsk = current;
2869
2870 plug->magic = PLUG_MAGIC;
2871 INIT_LIST_HEAD(&plug->list);
2872 INIT_LIST_HEAD(&plug->cb_list);
2873
2874
2875
2876
2877
2878 if (!tsk->plug) {
2879
2880
2881
2882
2883 tsk->plug = plug;
2884 }
2885}
2886EXPORT_SYMBOL(blk_start_plug);
2887
2888static int plug_rq_cmp(void *priv, struct list_head *a, struct list_head *b)
2889{
2890 struct request *rqa = container_of(a, struct request, queuelist);
2891 struct request *rqb = container_of(b, struct request, queuelist);
2892
2893 return !(rqa->q < rqb->q ||
2894 (rqa->q == rqb->q && blk_rq_pos(rqa) < blk_rq_pos(rqb)));
2895}
2896
2897
2898
2899
2900
2901
2902
2903static void queue_unplugged(struct request_queue *q, unsigned int depth,
2904 bool from_schedule)
2905 __releases(q->queue_lock)
2906{
2907 trace_block_unplug(q, depth, !from_schedule);
2908
2909 if (from_schedule)
2910 blk_run_queue_async(q);
2911 else
2912 __blk_run_queue(q);
2913 spin_unlock(q->queue_lock);
2914}
2915
2916static void flush_plug_callbacks(struct blk_plug *plug, bool from_schedule)
2917{
2918 LIST_HEAD(callbacks);
2919
2920 while (!list_empty(&plug->cb_list)) {
2921 list_splice_init(&plug->cb_list, &callbacks);
2922
2923 while (!list_empty(&callbacks)) {
2924 struct blk_plug_cb *cb = list_first_entry(&callbacks,
2925 struct blk_plug_cb,
2926 list);
2927 list_del(&cb->list);
2928 cb->callback(cb, from_schedule);
2929 }
2930 }
2931}
2932
2933struct blk_plug_cb *blk_check_plugged(blk_plug_cb_fn unplug, void *data,
2934 int size)
2935{
2936 struct blk_plug *plug = current->plug;
2937 struct blk_plug_cb *cb;
2938
2939 if (!plug)
2940 return NULL;
2941
2942 list_for_each_entry(cb, &plug->cb_list, list)
2943 if (cb->callback == unplug && cb->data == data)
2944 return cb;
2945
2946
2947 BUG_ON(size < sizeof(*cb));
2948 cb = kzalloc(size, GFP_ATOMIC);
2949 if (cb) {
2950 cb->data = data;
2951 cb->callback = unplug;
2952 list_add(&cb->list, &plug->cb_list);
2953 }
2954 return cb;
2955}
2956EXPORT_SYMBOL(blk_check_plugged);
2957
2958void blk_flush_plug_list(struct blk_plug *plug, bool from_schedule)
2959{
2960 struct request_queue *q;
2961 unsigned long flags;
2962 struct request *rq;
2963 LIST_HEAD(list);
2964 unsigned int depth;
2965
2966 BUG_ON(plug->magic != PLUG_MAGIC);
2967
2968 flush_plug_callbacks(plug, from_schedule);
2969 if (list_empty(&plug->list))
2970 return;
2971
2972 list_splice_init(&plug->list, &list);
2973
2974 list_sort(NULL, &list, plug_rq_cmp);
2975
2976 q = NULL;
2977 depth = 0;
2978
2979
2980
2981
2982
2983 local_irq_save(flags);
2984 while (!list_empty(&list)) {
2985 rq = list_entry_rq(list.next);
2986 list_del_init(&rq->queuelist);
2987 BUG_ON(!rq->q);
2988 if (rq->q != q) {
2989
2990
2991
2992 if (q)
2993 queue_unplugged(q, depth, from_schedule);
2994 q = rq->q;
2995 depth = 0;
2996 spin_lock(q->queue_lock);
2997 }
2998
2999
3000
3001
3002 if (unlikely(blk_queue_dying(q))) {
3003 __blk_end_request_all(rq, -ENODEV);
3004 continue;
3005 }
3006
3007
3008
3009
3010 if (rq->cmd_flags & (REQ_FLUSH | REQ_FUA))
3011 __elv_add_request(q, rq, ELEVATOR_INSERT_FLUSH);
3012 else
3013 __elv_add_request(q, rq, ELEVATOR_INSERT_SORT_MERGE);
3014
3015 depth++;
3016 }
3017
3018
3019
3020
3021 if (q)
3022 queue_unplugged(q, depth, from_schedule);
3023
3024 local_irq_restore(flags);
3025}
3026
3027void blk_finish_plug(struct blk_plug *plug)
3028{
3029 blk_flush_plug_list(plug, false);
3030
3031 if (plug == current->plug)
3032 current->plug = NULL;
3033}
3034EXPORT_SYMBOL(blk_finish_plug);
3035
3036#ifdef CONFIG_PM_RUNTIME
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058void blk_pm_runtime_init(struct request_queue *q, struct device *dev)
3059{
3060 q->dev = dev;
3061 q->rpm_status = RPM_ACTIVE;
3062 pm_runtime_set_autosuspend_delay(q->dev, -1);
3063 pm_runtime_use_autosuspend(q->dev);
3064}
3065EXPORT_SYMBOL(blk_pm_runtime_init);
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088int blk_pre_runtime_suspend(struct request_queue *q)
3089{
3090 int ret = 0;
3091
3092 spin_lock_irq(q->queue_lock);
3093 if (q->nr_pending) {
3094 ret = -EBUSY;
3095 pm_runtime_mark_last_busy(q->dev);
3096 } else {
3097 q->rpm_status = RPM_SUSPENDING;
3098 }
3099 spin_unlock_irq(q->queue_lock);
3100 return ret;
3101}
3102EXPORT_SYMBOL(blk_pre_runtime_suspend);
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117void blk_post_runtime_suspend(struct request_queue *q, int err)
3118{
3119 spin_lock_irq(q->queue_lock);
3120 if (!err) {
3121 q->rpm_status = RPM_SUSPENDED;
3122 } else {
3123 q->rpm_status = RPM_ACTIVE;
3124 pm_runtime_mark_last_busy(q->dev);
3125 }
3126 spin_unlock_irq(q->queue_lock);
3127}
3128EXPORT_SYMBOL(blk_post_runtime_suspend);
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141void blk_pre_runtime_resume(struct request_queue *q)
3142{
3143 spin_lock_irq(q->queue_lock);
3144 q->rpm_status = RPM_RESUMING;
3145 spin_unlock_irq(q->queue_lock);
3146}
3147EXPORT_SYMBOL(blk_pre_runtime_resume);
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163void blk_post_runtime_resume(struct request_queue *q, int err)
3164{
3165 spin_lock_irq(q->queue_lock);
3166 if (!err) {
3167 q->rpm_status = RPM_ACTIVE;
3168 __blk_run_queue(q);
3169 pm_runtime_mark_last_busy(q->dev);
3170 pm_request_autosuspend(q->dev);
3171 } else {
3172 q->rpm_status = RPM_SUSPENDED;
3173 }
3174 spin_unlock_irq(q->queue_lock);
3175}
3176EXPORT_SYMBOL(blk_post_runtime_resume);
3177#endif
3178
3179int __init blk_dev_init(void)
3180{
3181 BUILD_BUG_ON(__REQ_NR_BITS > 8 *
3182 sizeof(((struct request *)0)->cmd_flags));
3183
3184
3185 kblockd_workqueue = alloc_workqueue("kblockd",
3186 WQ_MEM_RECLAIM | WQ_HIGHPRI |
3187 WQ_POWER_EFFICIENT, 0);
3188 if (!kblockd_workqueue)
3189 panic("Failed to create kblockd\n");
3190
3191 request_cachep = kmem_cache_create("blkdev_requests",
3192 sizeof(struct request), 0, SLAB_PANIC, NULL);
3193
3194 blk_requestq_cachep = kmem_cache_create("blkdev_queue",
3195 sizeof(struct request_queue), 0, SLAB_PANIC, NULL);
3196
3197 return 0;
3198}
3199