1
2
3
4
5
6
7
8
9
10
11#include <linux/init.h>
12#include <linux/initrd.h>
13#include <linux/module.h>
14#include <linux/moduleparam.h>
15#include <linux/major.h>
16#include <linux/blkdev.h>
17#include <linux/bio.h>
18#include <linux/highmem.h>
19#include <linux/mutex.h>
20#include <linux/radix-tree.h>
21#include <linux/fs.h>
22#include <linux/slab.h>
23#ifdef CONFIG_BLK_DEV_RAM_DAX
24#include <linux/pfn_t.h>
25#include <linux/dax.h>
26#include <linux/uio.h>
27#endif
28
29#include <linux/uaccess.h>
30
31#define SECTOR_SHIFT 9
32#define PAGE_SECTORS_SHIFT (PAGE_SHIFT - SECTOR_SHIFT)
33#define PAGE_SECTORS (1 << PAGE_SECTORS_SHIFT)
34
35
36
37
38
39
40
41
42struct brd_device {
43 int brd_number;
44
45 struct request_queue *brd_queue;
46 struct gendisk *brd_disk;
47#ifdef CONFIG_BLK_DEV_RAM_DAX
48 struct dax_device *dax_dev;
49#endif
50 struct list_head brd_list;
51
52
53
54
55
56 spinlock_t brd_lock;
57 struct radix_tree_root brd_pages;
58};
59
60
61
62
63static DEFINE_MUTEX(brd_mutex);
64static struct page *brd_lookup_page(struct brd_device *brd, sector_t sector)
65{
66 pgoff_t idx;
67 struct page *page;
68
69
70
71
72
73
74
75
76
77
78
79
80 rcu_read_lock();
81 idx = sector >> PAGE_SECTORS_SHIFT;
82 page = radix_tree_lookup(&brd->brd_pages, idx);
83 rcu_read_unlock();
84
85 BUG_ON(page && page->index != idx);
86
87 return page;
88}
89
90
91
92
93
94
95static struct page *brd_insert_page(struct brd_device *brd, sector_t sector)
96{
97 pgoff_t idx;
98 struct page *page;
99 gfp_t gfp_flags;
100
101 page = brd_lookup_page(brd, sector);
102 if (page)
103 return page;
104
105
106
107
108
109
110
111
112
113
114 gfp_flags = GFP_NOIO | __GFP_ZERO;
115#ifndef CONFIG_BLK_DEV_RAM_DAX
116 gfp_flags |= __GFP_HIGHMEM;
117#endif
118 page = alloc_page(gfp_flags);
119 if (!page)
120 return NULL;
121
122 if (radix_tree_preload(GFP_NOIO)) {
123 __free_page(page);
124 return NULL;
125 }
126
127 spin_lock(&brd->brd_lock);
128 idx = sector >> PAGE_SECTORS_SHIFT;
129 page->index = idx;
130 if (radix_tree_insert(&brd->brd_pages, idx, page)) {
131 __free_page(page);
132 page = radix_tree_lookup(&brd->brd_pages, idx);
133 BUG_ON(!page);
134 BUG_ON(page->index != idx);
135 }
136 spin_unlock(&brd->brd_lock);
137
138 radix_tree_preload_end();
139
140 return page;
141}
142
143
144
145
146
147#define FREE_BATCH 16
148static void brd_free_pages(struct brd_device *brd)
149{
150 unsigned long pos = 0;
151 struct page *pages[FREE_BATCH];
152 int nr_pages;
153
154 do {
155 int i;
156
157 nr_pages = radix_tree_gang_lookup(&brd->brd_pages,
158 (void **)pages, pos, FREE_BATCH);
159
160 for (i = 0; i < nr_pages; i++) {
161 void *ret;
162
163 BUG_ON(pages[i]->index < pos);
164 pos = pages[i]->index;
165 ret = radix_tree_delete(&brd->brd_pages, pos);
166 BUG_ON(!ret || ret != pages[i]);
167 __free_page(pages[i]);
168 }
169
170 pos++;
171
172
173
174
175
176
177 } while (nr_pages == FREE_BATCH);
178}
179
180
181
182
183static int copy_to_brd_setup(struct brd_device *brd, sector_t sector, size_t n)
184{
185 unsigned int offset = (sector & (PAGE_SECTORS-1)) << SECTOR_SHIFT;
186 size_t copy;
187
188 copy = min_t(size_t, n, PAGE_SIZE - offset);
189 if (!brd_insert_page(brd, sector))
190 return -ENOSPC;
191 if (copy < n) {
192 sector += copy >> SECTOR_SHIFT;
193 if (!brd_insert_page(brd, sector))
194 return -ENOSPC;
195 }
196 return 0;
197}
198
199
200
201
202static void copy_to_brd(struct brd_device *brd, const void *src,
203 sector_t sector, size_t n)
204{
205 struct page *page;
206 void *dst;
207 unsigned int offset = (sector & (PAGE_SECTORS-1)) << SECTOR_SHIFT;
208 size_t copy;
209
210 copy = min_t(size_t, n, PAGE_SIZE - offset);
211 page = brd_lookup_page(brd, sector);
212 BUG_ON(!page);
213
214 dst = kmap_atomic(page);
215 memcpy(dst + offset, src, copy);
216 kunmap_atomic(dst);
217
218 if (copy < n) {
219 src += copy;
220 sector += copy >> SECTOR_SHIFT;
221 copy = n - copy;
222 page = brd_lookup_page(brd, sector);
223 BUG_ON(!page);
224
225 dst = kmap_atomic(page);
226 memcpy(dst, src, copy);
227 kunmap_atomic(dst);
228 }
229}
230
231
232
233
234static void copy_from_brd(void *dst, struct brd_device *brd,
235 sector_t sector, size_t n)
236{
237 struct page *page;
238 void *src;
239 unsigned int offset = (sector & (PAGE_SECTORS-1)) << SECTOR_SHIFT;
240 size_t copy;
241
242 copy = min_t(size_t, n, PAGE_SIZE - offset);
243 page = brd_lookup_page(brd, sector);
244 if (page) {
245 src = kmap_atomic(page);
246 memcpy(dst, src + offset, copy);
247 kunmap_atomic(src);
248 } else
249 memset(dst, 0, copy);
250
251 if (copy < n) {
252 dst += copy;
253 sector += copy >> SECTOR_SHIFT;
254 copy = n - copy;
255 page = brd_lookup_page(brd, sector);
256 if (page) {
257 src = kmap_atomic(page);
258 memcpy(dst, src, copy);
259 kunmap_atomic(src);
260 } else
261 memset(dst, 0, copy);
262 }
263}
264
265
266
267
268static int brd_do_bvec(struct brd_device *brd, struct page *page,
269 unsigned int len, unsigned int off, bool is_write,
270 sector_t sector)
271{
272 void *mem;
273 int err = 0;
274
275 if (is_write) {
276 err = copy_to_brd_setup(brd, sector, len);
277 if (err)
278 goto out;
279 }
280
281 mem = kmap_atomic(page);
282 if (!is_write) {
283 copy_from_brd(mem + off, brd, sector, len);
284 flush_dcache_page(page);
285 } else {
286 flush_dcache_page(page);
287 copy_to_brd(brd, mem + off, sector, len);
288 }
289 kunmap_atomic(mem);
290
291out:
292 return err;
293}
294
295static blk_qc_t brd_make_request(struct request_queue *q, struct bio *bio)
296{
297 struct brd_device *brd = bio->bi_disk->private_data;
298 struct bio_vec bvec;
299 sector_t sector;
300 struct bvec_iter iter;
301
302 sector = bio->bi_iter.bi_sector;
303 if (bio_end_sector(bio) > get_capacity(bio->bi_disk))
304 goto io_error;
305
306 bio_for_each_segment(bvec, bio, iter) {
307 unsigned int len = bvec.bv_len;
308 int err;
309
310 err = brd_do_bvec(brd, bvec.bv_page, len, bvec.bv_offset,
311 op_is_write(bio_op(bio)), sector);
312 if (err)
313 goto io_error;
314 sector += len >> SECTOR_SHIFT;
315 }
316
317 bio_endio(bio);
318 return BLK_QC_T_NONE;
319io_error:
320 bio_io_error(bio);
321 return BLK_QC_T_NONE;
322}
323
324static int brd_rw_page(struct block_device *bdev, sector_t sector,
325 struct page *page, bool is_write)
326{
327 struct brd_device *brd = bdev->bd_disk->private_data;
328 int err;
329
330 if (PageTransHuge(page))
331 return -ENOTSUPP;
332 err = brd_do_bvec(brd, page, PAGE_SIZE, 0, is_write, sector);
333 page_endio(page, is_write, err);
334 return err;
335}
336
337#ifdef CONFIG_BLK_DEV_RAM_DAX
338static long __brd_direct_access(struct brd_device *brd, pgoff_t pgoff,
339 long nr_pages, void **kaddr, pfn_t *pfn)
340{
341 struct page *page;
342
343 if (!brd)
344 return -ENODEV;
345 page = brd_insert_page(brd, (sector_t)pgoff << PAGE_SECTORS_SHIFT);
346 if (!page)
347 return -ENOSPC;
348 *kaddr = page_address(page);
349 *pfn = page_to_pfn_t(page);
350
351 return 1;
352}
353
354static long brd_dax_direct_access(struct dax_device *dax_dev,
355 pgoff_t pgoff, long nr_pages, void **kaddr, pfn_t *pfn)
356{
357 struct brd_device *brd = dax_get_private(dax_dev);
358
359 return __brd_direct_access(brd, pgoff, nr_pages, kaddr, pfn);
360}
361
362static size_t brd_dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff,
363 void *addr, size_t bytes, struct iov_iter *i)
364{
365 return copy_from_iter(addr, bytes, i);
366}
367
368static const struct dax_operations brd_dax_ops = {
369 .direct_access = brd_dax_direct_access,
370 .copy_from_iter = brd_dax_copy_from_iter,
371};
372#endif
373
374static const struct block_device_operations brd_fops = {
375 .owner = THIS_MODULE,
376 .rw_page = brd_rw_page,
377};
378
379
380
381
382static int rd_nr = CONFIG_BLK_DEV_RAM_COUNT;
383module_param(rd_nr, int, S_IRUGO);
384MODULE_PARM_DESC(rd_nr, "Maximum number of brd devices");
385
386unsigned long rd_size = CONFIG_BLK_DEV_RAM_SIZE;
387module_param(rd_size, ulong, S_IRUGO);
388MODULE_PARM_DESC(rd_size, "Size of each RAM disk in kbytes.");
389
390static int max_part = 1;
391module_param(max_part, int, S_IRUGO);
392MODULE_PARM_DESC(max_part, "Num Minors to reserve between devices");
393
394MODULE_LICENSE("GPL");
395MODULE_ALIAS_BLOCKDEV_MAJOR(RAMDISK_MAJOR);
396MODULE_ALIAS("rd");
397
398#ifndef MODULE
399
400static int __init ramdisk_size(char *str)
401{
402 rd_size = simple_strtol(str, NULL, 0);
403 return 1;
404}
405__setup("ramdisk_size=", ramdisk_size);
406#endif
407
408
409
410
411
412static LIST_HEAD(brd_devices);
413static DEFINE_MUTEX(brd_devices_mutex);
414
415static struct brd_device *brd_alloc(int i)
416{
417 struct brd_device *brd;
418 struct gendisk *disk;
419
420 brd = kzalloc(sizeof(*brd), GFP_KERNEL);
421 if (!brd)
422 goto out;
423 brd->brd_number = i;
424 spin_lock_init(&brd->brd_lock);
425 INIT_RADIX_TREE(&brd->brd_pages, GFP_ATOMIC);
426
427 brd->brd_queue = blk_alloc_queue(GFP_KERNEL);
428 if (!brd->brd_queue)
429 goto out_free_dev;
430
431 blk_queue_make_request(brd->brd_queue, brd_make_request);
432 blk_queue_max_hw_sectors(brd->brd_queue, 1024);
433
434
435
436
437
438
439
440 blk_queue_physical_block_size(brd->brd_queue, PAGE_SIZE);
441 disk = brd->brd_disk = alloc_disk(max_part);
442 if (!disk)
443 goto out_free_queue;
444 disk->major = RAMDISK_MAJOR;
445 disk->first_minor = i * max_part;
446 disk->fops = &brd_fops;
447 disk->private_data = brd;
448 disk->queue = brd->brd_queue;
449 disk->flags = GENHD_FL_EXT_DEVT;
450 sprintf(disk->disk_name, "ram%d", i);
451 set_capacity(disk, rd_size * 2);
452
453#ifdef CONFIG_BLK_DEV_RAM_DAX
454 queue_flag_set_unlocked(QUEUE_FLAG_DAX, brd->brd_queue);
455 brd->dax_dev = alloc_dax(brd, disk->disk_name, &brd_dax_ops);
456 if (!brd->dax_dev)
457 goto out_free_inode;
458#endif
459
460
461 return brd;
462
463#ifdef CONFIG_BLK_DEV_RAM_DAX
464out_free_inode:
465 kill_dax(brd->dax_dev);
466 put_dax(brd->dax_dev);
467#endif
468out_free_queue:
469 blk_cleanup_queue(brd->brd_queue);
470out_free_dev:
471 kfree(brd);
472out:
473 return NULL;
474}
475
476static void brd_free(struct brd_device *brd)
477{
478 put_disk(brd->brd_disk);
479 blk_cleanup_queue(brd->brd_queue);
480 brd_free_pages(brd);
481 kfree(brd);
482}
483
484static struct brd_device *brd_init_one(int i, bool *new)
485{
486 struct brd_device *brd;
487
488 *new = false;
489 list_for_each_entry(brd, &brd_devices, brd_list) {
490 if (brd->brd_number == i)
491 goto out;
492 }
493
494 brd = brd_alloc(i);
495 if (brd) {
496 add_disk(brd->brd_disk);
497 list_add_tail(&brd->brd_list, &brd_devices);
498 }
499 *new = true;
500out:
501 return brd;
502}
503
504static void brd_del_one(struct brd_device *brd)
505{
506 list_del(&brd->brd_list);
507#ifdef CONFIG_BLK_DEV_RAM_DAX
508 kill_dax(brd->dax_dev);
509 put_dax(brd->dax_dev);
510#endif
511 del_gendisk(brd->brd_disk);
512 brd_free(brd);
513}
514
515static struct kobject *brd_probe(dev_t dev, int *part, void *data)
516{
517 struct brd_device *brd;
518 struct kobject *kobj;
519 bool new;
520
521 mutex_lock(&brd_devices_mutex);
522 brd = brd_init_one(MINOR(dev) / max_part, &new);
523 kobj = brd ? get_disk(brd->brd_disk) : NULL;
524 mutex_unlock(&brd_devices_mutex);
525
526 if (new)
527 *part = 0;
528
529 return kobj;
530}
531
532static int __init brd_init(void)
533{
534 struct brd_device *brd, *next;
535 int i;
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552 if (register_blkdev(RAMDISK_MAJOR, "ramdisk"))
553 return -EIO;
554
555 if (unlikely(!max_part))
556 max_part = 1;
557
558 for (i = 0; i < rd_nr; i++) {
559 brd = brd_alloc(i);
560 if (!brd)
561 goto out_free;
562 list_add_tail(&brd->brd_list, &brd_devices);
563 }
564
565
566
567 list_for_each_entry(brd, &brd_devices, brd_list)
568 add_disk(brd->brd_disk);
569
570 blk_register_region(MKDEV(RAMDISK_MAJOR, 0), 1UL << MINORBITS,
571 THIS_MODULE, brd_probe, NULL, NULL);
572
573 pr_info("brd: module loaded\n");
574 return 0;
575
576out_free:
577 list_for_each_entry_safe(brd, next, &brd_devices, brd_list) {
578 list_del(&brd->brd_list);
579 brd_free(brd);
580 }
581 unregister_blkdev(RAMDISK_MAJOR, "ramdisk");
582
583 pr_info("brd: module NOT loaded !!!\n");
584 return -ENOMEM;
585}
586
587static void __exit brd_exit(void)
588{
589 struct brd_device *brd, *next;
590
591 list_for_each_entry_safe(brd, next, &brd_devices, brd_list)
592 brd_del_one(brd);
593
594 blk_unregister_region(MKDEV(RAMDISK_MAJOR, 0), 1UL << MINORBITS);
595 unregister_blkdev(RAMDISK_MAJOR, "ramdisk");
596
597 pr_info("brd: module unloaded\n");
598}
599
600module_init(brd_init);
601module_exit(brd_exit);
602
603