1
2
3
4
5#include <linux/kernel.h>
6#include <linux/slab.h>
7#include <linux/module.h>
8#include <linux/bio.h>
9#include <linux/blkdev.h>
10#include <linux/backing-dev.h>
11#include <linux/blktrace_api.h>
12#include <linux/blk-mq.h>
13#include <linux/blk-cgroup.h>
14#include <linux/debugfs.h>
15
16#include "blk.h"
17#include "blk-mq.h"
18#include "blk-mq-debugfs.h"
19#include "blk-wbt.h"
20
21struct queue_sysfs_entry {
22 struct attribute attr;
23 ssize_t (*show)(struct request_queue *, char *);
24 ssize_t (*store)(struct request_queue *, const char *, size_t);
25};
26
27static ssize_t
28queue_var_show(unsigned long var, char *page)
29{
30 return sprintf(page, "%lu\n", var);
31}
32
33static ssize_t
34queue_var_store(unsigned long *var, const char *page, size_t count)
35{
36 int err;
37 unsigned long v;
38
39 err = kstrtoul(page, 10, &v);
40 if (err || v > UINT_MAX)
41 return -EINVAL;
42
43 *var = v;
44
45 return count;
46}
47
48static ssize_t queue_var_store64(s64 *var, const char *page)
49{
50 int err;
51 s64 v;
52
53 err = kstrtos64(page, 10, &v);
54 if (err < 0)
55 return err;
56
57 *var = v;
58 return 0;
59}
60
61static ssize_t queue_requests_show(struct request_queue *q, char *page)
62{
63 return queue_var_show(q->nr_requests, page);
64}
65
66static ssize_t
67queue_requests_store(struct request_queue *q, const char *page, size_t count)
68{
69 unsigned long nr;
70 int ret, err;
71
72 if (!queue_is_mq(q))
73 return -EINVAL;
74
75 ret = queue_var_store(&nr, page, count);
76 if (ret < 0)
77 return ret;
78
79 if (nr < BLKDEV_MIN_RQ)
80 nr = BLKDEV_MIN_RQ;
81
82 err = blk_mq_update_nr_requests(q, nr);
83 if (err)
84 return err;
85
86 return ret;
87}
88
89static ssize_t queue_ra_show(struct request_queue *q, char *page)
90{
91 unsigned long ra_kb;
92
93 if (!q->disk)
94 return -EINVAL;
95 ra_kb = q->disk->bdi->ra_pages << (PAGE_SHIFT - 10);
96 return queue_var_show(ra_kb, page);
97}
98
99static ssize_t
100queue_ra_store(struct request_queue *q, const char *page, size_t count)
101{
102 unsigned long ra_kb;
103 ssize_t ret;
104
105 if (!q->disk)
106 return -EINVAL;
107 ret = queue_var_store(&ra_kb, page, count);
108 if (ret < 0)
109 return ret;
110 q->disk->bdi->ra_pages = ra_kb >> (PAGE_SHIFT - 10);
111 return ret;
112}
113
114static ssize_t queue_max_sectors_show(struct request_queue *q, char *page)
115{
116 int max_sectors_kb = queue_max_sectors(q) >> 1;
117
118 return queue_var_show(max_sectors_kb, page);
119}
120
121static ssize_t queue_max_segments_show(struct request_queue *q, char *page)
122{
123 return queue_var_show(queue_max_segments(q), page);
124}
125
126static ssize_t queue_max_discard_segments_show(struct request_queue *q,
127 char *page)
128{
129 return queue_var_show(queue_max_discard_segments(q), page);
130}
131
132static ssize_t queue_max_integrity_segments_show(struct request_queue *q, char *page)
133{
134 return queue_var_show(q->limits.max_integrity_segments, page);
135}
136
137static ssize_t queue_max_segment_size_show(struct request_queue *q, char *page)
138{
139 return queue_var_show(queue_max_segment_size(q), page);
140}
141
142static ssize_t queue_logical_block_size_show(struct request_queue *q, char *page)
143{
144 return queue_var_show(queue_logical_block_size(q), page);
145}
146
147static ssize_t queue_physical_block_size_show(struct request_queue *q, char *page)
148{
149 return queue_var_show(queue_physical_block_size(q), page);
150}
151
152static ssize_t queue_chunk_sectors_show(struct request_queue *q, char *page)
153{
154 return queue_var_show(q->limits.chunk_sectors, page);
155}
156
157static ssize_t queue_io_min_show(struct request_queue *q, char *page)
158{
159 return queue_var_show(queue_io_min(q), page);
160}
161
162static ssize_t queue_io_opt_show(struct request_queue *q, char *page)
163{
164 return queue_var_show(queue_io_opt(q), page);
165}
166
167static ssize_t queue_discard_granularity_show(struct request_queue *q, char *page)
168{
169 return queue_var_show(q->limits.discard_granularity, page);
170}
171
172static ssize_t queue_discard_max_hw_show(struct request_queue *q, char *page)
173{
174
175 return sprintf(page, "%llu\n",
176 (unsigned long long)q->limits.max_hw_discard_sectors << 9);
177}
178
179static ssize_t queue_discard_max_show(struct request_queue *q, char *page)
180{
181 return sprintf(page, "%llu\n",
182 (unsigned long long)q->limits.max_discard_sectors << 9);
183}
184
185static ssize_t queue_discard_max_store(struct request_queue *q,
186 const char *page, size_t count)
187{
188 unsigned long max_discard;
189 ssize_t ret = queue_var_store(&max_discard, page, count);
190
191 if (ret < 0)
192 return ret;
193
194 if (max_discard & (q->limits.discard_granularity - 1))
195 return -EINVAL;
196
197 max_discard >>= 9;
198 if (max_discard > UINT_MAX)
199 return -EINVAL;
200
201 if (max_discard > q->limits.max_hw_discard_sectors)
202 max_discard = q->limits.max_hw_discard_sectors;
203
204 q->limits.max_discard_sectors = max_discard;
205 return ret;
206}
207
208static ssize_t queue_discard_zeroes_data_show(struct request_queue *q, char *page)
209{
210 return queue_var_show(0, page);
211}
212
213static ssize_t queue_write_same_max_show(struct request_queue *q, char *page)
214{
215 return sprintf(page, "%llu\n",
216 (unsigned long long)q->limits.max_write_same_sectors << 9);
217}
218
219static ssize_t queue_write_zeroes_max_show(struct request_queue *q, char *page)
220{
221 return sprintf(page, "%llu\n",
222 (unsigned long long)q->limits.max_write_zeroes_sectors << 9);
223}
224
225static ssize_t queue_zone_write_granularity_show(struct request_queue *q,
226 char *page)
227{
228 return queue_var_show(queue_zone_write_granularity(q), page);
229}
230
231static ssize_t queue_zone_append_max_show(struct request_queue *q, char *page)
232{
233 unsigned long long max_sectors = q->limits.max_zone_append_sectors;
234
235 return sprintf(page, "%llu\n", max_sectors << SECTOR_SHIFT);
236}
237
238static ssize_t
239queue_max_sectors_store(struct request_queue *q, const char *page, size_t count)
240{
241 unsigned long max_sectors_kb,
242 max_hw_sectors_kb = queue_max_hw_sectors(q) >> 1,
243 page_kb = 1 << (PAGE_SHIFT - 10);
244 ssize_t ret = queue_var_store(&max_sectors_kb, page, count);
245
246 if (ret < 0)
247 return ret;
248
249 max_hw_sectors_kb = min_not_zero(max_hw_sectors_kb, (unsigned long)
250 q->limits.max_dev_sectors >> 1);
251
252 if (max_sectors_kb > max_hw_sectors_kb || max_sectors_kb < page_kb)
253 return -EINVAL;
254
255 spin_lock_irq(&q->queue_lock);
256 q->limits.max_sectors = max_sectors_kb << 1;
257 if (q->disk)
258 q->disk->bdi->io_pages = max_sectors_kb >> (PAGE_SHIFT - 10);
259 spin_unlock_irq(&q->queue_lock);
260
261 return ret;
262}
263
264static ssize_t queue_max_hw_sectors_show(struct request_queue *q, char *page)
265{
266 int max_hw_sectors_kb = queue_max_hw_sectors(q) >> 1;
267
268 return queue_var_show(max_hw_sectors_kb, page);
269}
270
271static ssize_t queue_virt_boundary_mask_show(struct request_queue *q, char *page)
272{
273 return queue_var_show(q->limits.virt_boundary_mask, page);
274}
275
276#define QUEUE_SYSFS_BIT_FNS(name, flag, neg) \
277static ssize_t \
278queue_##name##_show(struct request_queue *q, char *page) \
279{ \
280 int bit; \
281 bit = test_bit(QUEUE_FLAG_##flag, &q->queue_flags); \
282 return queue_var_show(neg ? !bit : bit, page); \
283} \
284static ssize_t \
285queue_##name##_store(struct request_queue *q, const char *page, size_t count) \
286{ \
287 unsigned long val; \
288 ssize_t ret; \
289 ret = queue_var_store(&val, page, count); \
290 if (ret < 0) \
291 return ret; \
292 if (neg) \
293 val = !val; \
294 \
295 if (val) \
296 blk_queue_flag_set(QUEUE_FLAG_##flag, q); \
297 else \
298 blk_queue_flag_clear(QUEUE_FLAG_##flag, q); \
299 return ret; \
300}
301
302QUEUE_SYSFS_BIT_FNS(nonrot, NONROT, 1);
303QUEUE_SYSFS_BIT_FNS(random, ADD_RANDOM, 0);
304QUEUE_SYSFS_BIT_FNS(iostats, IO_STAT, 0);
305QUEUE_SYSFS_BIT_FNS(stable_writes, STABLE_WRITES, 0);
306#undef QUEUE_SYSFS_BIT_FNS
307
308static ssize_t queue_zoned_show(struct request_queue *q, char *page)
309{
310 switch (blk_queue_zoned_model(q)) {
311 case BLK_ZONED_HA:
312 return sprintf(page, "host-aware\n");
313 case BLK_ZONED_HM:
314 return sprintf(page, "host-managed\n");
315 default:
316 return sprintf(page, "none\n");
317 }
318}
319
320static ssize_t queue_nr_zones_show(struct request_queue *q, char *page)
321{
322 return queue_var_show(blk_queue_nr_zones(q), page);
323}
324
325static ssize_t queue_max_open_zones_show(struct request_queue *q, char *page)
326{
327 return queue_var_show(queue_max_open_zones(q), page);
328}
329
330static ssize_t queue_max_active_zones_show(struct request_queue *q, char *page)
331{
332 return queue_var_show(queue_max_active_zones(q), page);
333}
334
335static ssize_t queue_nomerges_show(struct request_queue *q, char *page)
336{
337 return queue_var_show((blk_queue_nomerges(q) << 1) |
338 blk_queue_noxmerges(q), page);
339}
340
341static ssize_t queue_nomerges_store(struct request_queue *q, const char *page,
342 size_t count)
343{
344 unsigned long nm;
345 ssize_t ret = queue_var_store(&nm, page, count);
346
347 if (ret < 0)
348 return ret;
349
350 blk_queue_flag_clear(QUEUE_FLAG_NOMERGES, q);
351 blk_queue_flag_clear(QUEUE_FLAG_NOXMERGES, q);
352 if (nm == 2)
353 blk_queue_flag_set(QUEUE_FLAG_NOMERGES, q);
354 else if (nm)
355 blk_queue_flag_set(QUEUE_FLAG_NOXMERGES, q);
356
357 return ret;
358}
359
360static ssize_t queue_rq_affinity_show(struct request_queue *q, char *page)
361{
362 bool set = test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags);
363 bool force = test_bit(QUEUE_FLAG_SAME_FORCE, &q->queue_flags);
364
365 return queue_var_show(set << force, page);
366}
367
368static ssize_t
369queue_rq_affinity_store(struct request_queue *q, const char *page, size_t count)
370{
371 ssize_t ret = -EINVAL;
372#ifdef CONFIG_SMP
373 unsigned long val;
374
375 ret = queue_var_store(&val, page, count);
376 if (ret < 0)
377 return ret;
378
379 if (val == 2) {
380 blk_queue_flag_set(QUEUE_FLAG_SAME_COMP, q);
381 blk_queue_flag_set(QUEUE_FLAG_SAME_FORCE, q);
382 } else if (val == 1) {
383 blk_queue_flag_set(QUEUE_FLAG_SAME_COMP, q);
384 blk_queue_flag_clear(QUEUE_FLAG_SAME_FORCE, q);
385 } else if (val == 0) {
386 blk_queue_flag_clear(QUEUE_FLAG_SAME_COMP, q);
387 blk_queue_flag_clear(QUEUE_FLAG_SAME_FORCE, q);
388 }
389#endif
390 return ret;
391}
392
393static ssize_t queue_poll_delay_show(struct request_queue *q, char *page)
394{
395 int val;
396
397 if (q->poll_nsec == BLK_MQ_POLL_CLASSIC)
398 val = BLK_MQ_POLL_CLASSIC;
399 else
400 val = q->poll_nsec / 1000;
401
402 return sprintf(page, "%d\n", val);
403}
404
405static ssize_t queue_poll_delay_store(struct request_queue *q, const char *page,
406 size_t count)
407{
408 int err, val;
409
410 if (!q->mq_ops || !q->mq_ops->poll)
411 return -EINVAL;
412
413 err = kstrtoint(page, 10, &val);
414 if (err < 0)
415 return err;
416
417 if (val == BLK_MQ_POLL_CLASSIC)
418 q->poll_nsec = BLK_MQ_POLL_CLASSIC;
419 else if (val >= 0)
420 q->poll_nsec = val * 1000;
421 else
422 return -EINVAL;
423
424 return count;
425}
426
427static ssize_t queue_poll_show(struct request_queue *q, char *page)
428{
429 return queue_var_show(test_bit(QUEUE_FLAG_POLL, &q->queue_flags), page);
430}
431
432static ssize_t queue_poll_store(struct request_queue *q, const char *page,
433 size_t count)
434{
435 unsigned long poll_on;
436 ssize_t ret;
437
438 if (!q->tag_set || q->tag_set->nr_maps <= HCTX_TYPE_POLL ||
439 !q->tag_set->map[HCTX_TYPE_POLL].nr_queues)
440 return -EINVAL;
441
442 ret = queue_var_store(&poll_on, page, count);
443 if (ret < 0)
444 return ret;
445
446 if (poll_on) {
447 blk_queue_flag_set(QUEUE_FLAG_POLL, q);
448 } else {
449 blk_mq_freeze_queue(q);
450 blk_queue_flag_clear(QUEUE_FLAG_POLL, q);
451 blk_mq_unfreeze_queue(q);
452 }
453
454 return ret;
455}
456
457static ssize_t queue_io_timeout_show(struct request_queue *q, char *page)
458{
459 return sprintf(page, "%u\n", jiffies_to_msecs(q->rq_timeout));
460}
461
462static ssize_t queue_io_timeout_store(struct request_queue *q, const char *page,
463 size_t count)
464{
465 unsigned int val;
466 int err;
467
468 err = kstrtou32(page, 10, &val);
469 if (err || val == 0)
470 return -EINVAL;
471
472 blk_queue_rq_timeout(q, msecs_to_jiffies(val));
473
474 return count;
475}
476
477static ssize_t queue_wb_lat_show(struct request_queue *q, char *page)
478{
479 if (!wbt_rq_qos(q))
480 return -EINVAL;
481
482 return sprintf(page, "%llu\n", div_u64(wbt_get_min_lat(q), 1000));
483}
484
485static ssize_t queue_wb_lat_store(struct request_queue *q, const char *page,
486 size_t count)
487{
488 struct rq_qos *rqos;
489 ssize_t ret;
490 s64 val;
491
492 ret = queue_var_store64(&val, page);
493 if (ret < 0)
494 return ret;
495 if (val < -1)
496 return -EINVAL;
497
498 rqos = wbt_rq_qos(q);
499 if (!rqos) {
500 ret = wbt_init(q);
501 if (ret)
502 return ret;
503 }
504
505 if (val == -1)
506 val = wbt_default_latency_nsec(q);
507 else if (val >= 0)
508 val *= 1000ULL;
509
510 if (wbt_get_min_lat(q) == val)
511 return count;
512
513
514
515
516
517
518 blk_mq_freeze_queue(q);
519 blk_mq_quiesce_queue(q);
520
521 wbt_set_min_lat(q, val);
522
523 blk_mq_unquiesce_queue(q);
524 blk_mq_unfreeze_queue(q);
525
526 return count;
527}
528
529static ssize_t queue_wc_show(struct request_queue *q, char *page)
530{
531 if (test_bit(QUEUE_FLAG_WC, &q->queue_flags))
532 return sprintf(page, "write back\n");
533
534 return sprintf(page, "write through\n");
535}
536
537static ssize_t queue_wc_store(struct request_queue *q, const char *page,
538 size_t count)
539{
540 int set = -1;
541
542 if (!strncmp(page, "write back", 10))
543 set = 1;
544 else if (!strncmp(page, "write through", 13) ||
545 !strncmp(page, "none", 4))
546 set = 0;
547
548 if (set == -1)
549 return -EINVAL;
550
551 if (set)
552 blk_queue_flag_set(QUEUE_FLAG_WC, q);
553 else
554 blk_queue_flag_clear(QUEUE_FLAG_WC, q);
555
556 return count;
557}
558
559static ssize_t queue_fua_show(struct request_queue *q, char *page)
560{
561 return sprintf(page, "%u\n", test_bit(QUEUE_FLAG_FUA, &q->queue_flags));
562}
563
564static ssize_t queue_dax_show(struct request_queue *q, char *page)
565{
566 return queue_var_show(blk_queue_dax(q), page);
567}
568
569#define QUEUE_RO_ENTRY(_prefix, _name) \
570static struct queue_sysfs_entry _prefix##_entry = { \
571 .attr = { .name = _name, .mode = 0444 }, \
572 .show = _prefix##_show, \
573};
574
575#define QUEUE_RW_ENTRY(_prefix, _name) \
576static struct queue_sysfs_entry _prefix##_entry = { \
577 .attr = { .name = _name, .mode = 0644 }, \
578 .show = _prefix##_show, \
579 .store = _prefix##_store, \
580};
581
582QUEUE_RW_ENTRY(queue_requests, "nr_requests");
583QUEUE_RW_ENTRY(queue_ra, "read_ahead_kb");
584QUEUE_RW_ENTRY(queue_max_sectors, "max_sectors_kb");
585QUEUE_RO_ENTRY(queue_max_hw_sectors, "max_hw_sectors_kb");
586QUEUE_RO_ENTRY(queue_max_segments, "max_segments");
587QUEUE_RO_ENTRY(queue_max_integrity_segments, "max_integrity_segments");
588QUEUE_RO_ENTRY(queue_max_segment_size, "max_segment_size");
589QUEUE_RW_ENTRY(elv_iosched, "scheduler");
590
591QUEUE_RO_ENTRY(queue_logical_block_size, "logical_block_size");
592QUEUE_RO_ENTRY(queue_physical_block_size, "physical_block_size");
593QUEUE_RO_ENTRY(queue_chunk_sectors, "chunk_sectors");
594QUEUE_RO_ENTRY(queue_io_min, "minimum_io_size");
595QUEUE_RO_ENTRY(queue_io_opt, "optimal_io_size");
596
597QUEUE_RO_ENTRY(queue_max_discard_segments, "max_discard_segments");
598QUEUE_RO_ENTRY(queue_discard_granularity, "discard_granularity");
599QUEUE_RO_ENTRY(queue_discard_max_hw, "discard_max_hw_bytes");
600QUEUE_RW_ENTRY(queue_discard_max, "discard_max_bytes");
601QUEUE_RO_ENTRY(queue_discard_zeroes_data, "discard_zeroes_data");
602
603QUEUE_RO_ENTRY(queue_write_same_max, "write_same_max_bytes");
604QUEUE_RO_ENTRY(queue_write_zeroes_max, "write_zeroes_max_bytes");
605QUEUE_RO_ENTRY(queue_zone_append_max, "zone_append_max_bytes");
606QUEUE_RO_ENTRY(queue_zone_write_granularity, "zone_write_granularity");
607
608QUEUE_RO_ENTRY(queue_zoned, "zoned");
609QUEUE_RO_ENTRY(queue_nr_zones, "nr_zones");
610QUEUE_RO_ENTRY(queue_max_open_zones, "max_open_zones");
611QUEUE_RO_ENTRY(queue_max_active_zones, "max_active_zones");
612
613QUEUE_RW_ENTRY(queue_nomerges, "nomerges");
614QUEUE_RW_ENTRY(queue_rq_affinity, "rq_affinity");
615QUEUE_RW_ENTRY(queue_poll, "io_poll");
616QUEUE_RW_ENTRY(queue_poll_delay, "io_poll_delay");
617QUEUE_RW_ENTRY(queue_wc, "write_cache");
618QUEUE_RO_ENTRY(queue_fua, "fua");
619QUEUE_RO_ENTRY(queue_dax, "dax");
620QUEUE_RW_ENTRY(queue_io_timeout, "io_timeout");
621QUEUE_RW_ENTRY(queue_wb_lat, "wbt_lat_usec");
622QUEUE_RO_ENTRY(queue_virt_boundary_mask, "virt_boundary_mask");
623
624#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
625QUEUE_RW_ENTRY(blk_throtl_sample_time, "throttle_sample_time");
626#endif
627
628
629static struct queue_sysfs_entry queue_hw_sector_size_entry = {
630 .attr = {.name = "hw_sector_size", .mode = 0444 },
631 .show = queue_logical_block_size_show,
632};
633
634QUEUE_RW_ENTRY(queue_nonrot, "rotational");
635QUEUE_RW_ENTRY(queue_iostats, "iostats");
636QUEUE_RW_ENTRY(queue_random, "add_random");
637QUEUE_RW_ENTRY(queue_stable_writes, "stable_writes");
638
639static struct attribute *queue_attrs[] = {
640 &queue_requests_entry.attr,
641 &queue_ra_entry.attr,
642 &queue_max_hw_sectors_entry.attr,
643 &queue_max_sectors_entry.attr,
644 &queue_max_segments_entry.attr,
645 &queue_max_discard_segments_entry.attr,
646 &queue_max_integrity_segments_entry.attr,
647 &queue_max_segment_size_entry.attr,
648 &elv_iosched_entry.attr,
649 &queue_hw_sector_size_entry.attr,
650 &queue_logical_block_size_entry.attr,
651 &queue_physical_block_size_entry.attr,
652 &queue_chunk_sectors_entry.attr,
653 &queue_io_min_entry.attr,
654 &queue_io_opt_entry.attr,
655 &queue_discard_granularity_entry.attr,
656 &queue_discard_max_entry.attr,
657 &queue_discard_max_hw_entry.attr,
658 &queue_discard_zeroes_data_entry.attr,
659 &queue_write_same_max_entry.attr,
660 &queue_write_zeroes_max_entry.attr,
661 &queue_zone_append_max_entry.attr,
662 &queue_zone_write_granularity_entry.attr,
663 &queue_nonrot_entry.attr,
664 &queue_zoned_entry.attr,
665 &queue_nr_zones_entry.attr,
666 &queue_max_open_zones_entry.attr,
667 &queue_max_active_zones_entry.attr,
668 &queue_nomerges_entry.attr,
669 &queue_rq_affinity_entry.attr,
670 &queue_iostats_entry.attr,
671 &queue_stable_writes_entry.attr,
672 &queue_random_entry.attr,
673 &queue_poll_entry.attr,
674 &queue_wc_entry.attr,
675 &queue_fua_entry.attr,
676 &queue_dax_entry.attr,
677 &queue_wb_lat_entry.attr,
678 &queue_poll_delay_entry.attr,
679 &queue_io_timeout_entry.attr,
680#ifdef CONFIG_BLK_DEV_THROTTLING_LOW
681 &blk_throtl_sample_time_entry.attr,
682#endif
683 &queue_virt_boundary_mask_entry.attr,
684 NULL,
685};
686
687static umode_t queue_attr_visible(struct kobject *kobj, struct attribute *attr,
688 int n)
689{
690 struct request_queue *q =
691 container_of(kobj, struct request_queue, kobj);
692
693 if (attr == &queue_io_timeout_entry.attr &&
694 (!q->mq_ops || !q->mq_ops->timeout))
695 return 0;
696
697 if ((attr == &queue_max_open_zones_entry.attr ||
698 attr == &queue_max_active_zones_entry.attr) &&
699 !blk_queue_is_zoned(q))
700 return 0;
701
702 return attr->mode;
703}
704
705static struct attribute_group queue_attr_group = {
706 .attrs = queue_attrs,
707 .is_visible = queue_attr_visible,
708};
709
710
711#define to_queue(atr) container_of((atr), struct queue_sysfs_entry, attr)
712
713static ssize_t
714queue_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
715{
716 struct queue_sysfs_entry *entry = to_queue(attr);
717 struct request_queue *q =
718 container_of(kobj, struct request_queue, kobj);
719 ssize_t res;
720
721 if (!entry->show)
722 return -EIO;
723 mutex_lock(&q->sysfs_lock);
724 res = entry->show(q, page);
725 mutex_unlock(&q->sysfs_lock);
726 return res;
727}
728
729static ssize_t
730queue_attr_store(struct kobject *kobj, struct attribute *attr,
731 const char *page, size_t length)
732{
733 struct queue_sysfs_entry *entry = to_queue(attr);
734 struct request_queue *q;
735 ssize_t res;
736
737 if (!entry->store)
738 return -EIO;
739
740 q = container_of(kobj, struct request_queue, kobj);
741 mutex_lock(&q->sysfs_lock);
742 res = entry->store(q, page, length);
743 mutex_unlock(&q->sysfs_lock);
744 return res;
745}
746
747static void blk_free_queue_rcu(struct rcu_head *rcu_head)
748{
749 struct request_queue *q = container_of(rcu_head, struct request_queue,
750 rcu_head);
751 kmem_cache_free(blk_requestq_cachep, q);
752}
753
754
755static void blk_exit_queue(struct request_queue *q)
756{
757
758
759
760
761
762 if (q->elevator) {
763 ioc_clear_queue(q);
764 __elevator_exit(q, q->elevator);
765 }
766
767
768
769
770
771
772 blkcg_exit_queue(q);
773}
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795static void blk_release_queue(struct kobject *kobj)
796{
797 struct request_queue *q =
798 container_of(kobj, struct request_queue, kobj);
799
800 might_sleep();
801
802 if (test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags))
803 blk_stat_remove_callback(q, q->poll_cb);
804 blk_stat_free_callback(q->poll_cb);
805
806 blk_free_queue_stats(q->stats);
807
808 if (queue_is_mq(q)) {
809 struct blk_mq_hw_ctx *hctx;
810 int i;
811
812 cancel_delayed_work_sync(&q->requeue_work);
813
814 queue_for_each_hw_ctx(q, hctx, i)
815 cancel_delayed_work_sync(&hctx->run_work);
816 }
817
818 blk_exit_queue(q);
819
820 blk_queue_free_zone_bitmaps(q);
821
822 if (queue_is_mq(q))
823 blk_mq_release(q);
824
825 blk_trace_shutdown(q);
826 mutex_lock(&q->debugfs_mutex);
827 debugfs_remove_recursive(q->debugfs_dir);
828 mutex_unlock(&q->debugfs_mutex);
829
830 if (queue_is_mq(q))
831 blk_mq_debugfs_unregister(q);
832
833 bioset_exit(&q->bio_split);
834
835 ida_simple_remove(&blk_queue_ida, q->id);
836 call_rcu(&q->rcu_head, blk_free_queue_rcu);
837}
838
839static const struct sysfs_ops queue_sysfs_ops = {
840 .show = queue_attr_show,
841 .store = queue_attr_store,
842};
843
844struct kobj_type blk_queue_ktype = {
845 .sysfs_ops = &queue_sysfs_ops,
846 .release = blk_release_queue,
847};
848
849
850
851
852
853int blk_register_queue(struct gendisk *disk)
854{
855 int ret;
856 struct device *dev = disk_to_dev(disk);
857 struct request_queue *q = disk->queue;
858
859 ret = blk_trace_init_sysfs(dev);
860 if (ret)
861 return ret;
862
863 mutex_lock(&q->sysfs_dir_lock);
864
865 ret = kobject_add(&q->kobj, kobject_get(&dev->kobj), "%s", "queue");
866 if (ret < 0) {
867 blk_trace_remove_sysfs(dev);
868 goto unlock;
869 }
870
871 ret = sysfs_create_group(&q->kobj, &queue_attr_group);
872 if (ret) {
873 blk_trace_remove_sysfs(dev);
874 kobject_del(&q->kobj);
875 kobject_put(&dev->kobj);
876 goto unlock;
877 }
878
879 mutex_lock(&q->debugfs_mutex);
880 q->debugfs_dir = debugfs_create_dir(kobject_name(q->kobj.parent),
881 blk_debugfs_root);
882 mutex_unlock(&q->debugfs_mutex);
883
884 if (queue_is_mq(q)) {
885 __blk_mq_register_dev(dev, q);
886 blk_mq_debugfs_register(q);
887 }
888
889 mutex_lock(&q->sysfs_lock);
890 if (q->elevator) {
891 ret = elv_register_queue(q, false);
892 if (ret) {
893 mutex_unlock(&q->sysfs_lock);
894 mutex_unlock(&q->sysfs_dir_lock);
895 kobject_del(&q->kobj);
896 blk_trace_remove_sysfs(dev);
897 kobject_put(&dev->kobj);
898 return ret;
899 }
900 }
901
902 blk_queue_flag_set(QUEUE_FLAG_REGISTERED, q);
903 wbt_enable_default(q);
904 blk_throtl_register_queue(q);
905
906
907 kobject_uevent(&q->kobj, KOBJ_ADD);
908 if (q->elevator)
909 kobject_uevent(&q->elevator->kobj, KOBJ_ADD);
910 mutex_unlock(&q->sysfs_lock);
911
912 ret = 0;
913unlock:
914 mutex_unlock(&q->sysfs_dir_lock);
915
916
917
918
919
920
921
922
923
924
925 if (!blk_queue_init_done(q)) {
926 blk_queue_flag_set(QUEUE_FLAG_INIT_DONE, q);
927 percpu_ref_switch_to_percpu(&q->q_usage_counter);
928 }
929
930 return ret;
931}
932
933
934
935
936
937
938
939
940void blk_unregister_queue(struct gendisk *disk)
941{
942 struct request_queue *q = disk->queue;
943
944 if (WARN_ON(!q))
945 return;
946
947
948 if (!blk_queue_registered(q))
949 return;
950
951
952
953
954
955
956 mutex_lock(&q->sysfs_lock);
957 blk_queue_flag_clear(QUEUE_FLAG_REGISTERED, q);
958 mutex_unlock(&q->sysfs_lock);
959
960 mutex_lock(&q->sysfs_dir_lock);
961
962
963
964
965 if (queue_is_mq(q))
966 blk_mq_unregister_dev(disk_to_dev(disk), q);
967
968 kobject_uevent(&q->kobj, KOBJ_REMOVE);
969 kobject_del(&q->kobj);
970 blk_trace_remove_sysfs(disk_to_dev(disk));
971
972 mutex_lock(&q->sysfs_lock);
973 if (q->elevator)
974 elv_unregister_queue(q);
975 mutex_unlock(&q->sysfs_lock);
976 mutex_unlock(&q->sysfs_dir_lock);
977
978 kobject_put(&disk_to_dev(disk)->kobj);
979}
980