1
2
3
4
5
6
7
8#include <linux/init.h>
9#include <linux/mm.h>
10#include <linux/fcntl.h>
11#include <linux/slab.h>
12#include <linux/kmod.h>
13#include <linux/major.h>
14#include <linux/device_cgroup.h>
15#include <linux/highmem.h>
16#include <linux/blkdev.h>
17#include <linux/backing-dev.h>
18#include <linux/module.h>
19#include <linux/blkpg.h>
20#include <linux/magic.h>
21#include <linux/dax.h>
22#include <linux/buffer_head.h>
23#include <linux/swap.h>
24#include <linux/pagevec.h>
25#include <linux/writeback.h>
26#include <linux/mpage.h>
27#include <linux/mount.h>
28#include <linux/uio.h>
29#include <linux/namei.h>
30#include <linux/log2.h>
31#include <linux/cleancache.h>
32#include <linux/dax.h>
33#include <linux/badblocks.h>
34#include <linux/task_io_accounting_ops.h>
35#include <linux/falloc.h>
36#include <linux/uaccess.h>
37#include "internal.h"
38
39struct bdev_inode {
40 struct block_device bdev;
41 struct inode vfs_inode;
42};
43
44static const struct address_space_operations def_blk_aops;
45
46static inline struct bdev_inode *BDEV_I(struct inode *inode)
47{
48 return container_of(inode, struct bdev_inode, vfs_inode);
49}
50
51struct block_device *I_BDEV(struct inode *inode)
52{
53 return &BDEV_I(inode)->bdev;
54}
55EXPORT_SYMBOL(I_BDEV);
56
57static void bdev_write_inode(struct block_device *bdev)
58{
59 struct inode *inode = bdev->bd_inode;
60 int ret;
61
62 spin_lock(&inode->i_lock);
63 while (inode->i_state & I_DIRTY) {
64 spin_unlock(&inode->i_lock);
65 ret = write_inode_now(inode, true);
66 if (ret) {
67 char name[BDEVNAME_SIZE];
68 pr_warn_ratelimited("VFS: Dirty inode writeback failed "
69 "for block device %s (err=%d).\n",
70 bdevname(bdev, name), ret);
71 }
72 spin_lock(&inode->i_lock);
73 }
74 spin_unlock(&inode->i_lock);
75}
76
77
78void kill_bdev(struct block_device *bdev)
79{
80 struct address_space *mapping = bdev->bd_inode->i_mapping;
81
82 if (mapping->nrpages == 0 && mapping->nrexceptional == 0)
83 return;
84
85 invalidate_bh_lrus();
86 truncate_inode_pages(mapping, 0);
87}
88EXPORT_SYMBOL(kill_bdev);
89
90
91void invalidate_bdev(struct block_device *bdev)
92{
93 struct address_space *mapping = bdev->bd_inode->i_mapping;
94
95 if (mapping->nrpages) {
96 invalidate_bh_lrus();
97 lru_add_drain_all();
98 invalidate_mapping_pages(mapping, 0, -1);
99 }
100
101
102
103 cleancache_invalidate_inode(mapping);
104}
105EXPORT_SYMBOL(invalidate_bdev);
106
107int set_blocksize(struct block_device *bdev, int size)
108{
109
110 if (size > PAGE_SIZE || size < 512 || !is_power_of_2(size))
111 return -EINVAL;
112
113
114 if (size < bdev_logical_block_size(bdev))
115 return -EINVAL;
116
117
118 if (bdev->bd_block_size != size) {
119 sync_blockdev(bdev);
120 bdev->bd_block_size = size;
121 bdev->bd_inode->i_blkbits = blksize_bits(size);
122 kill_bdev(bdev);
123 }
124 return 0;
125}
126
127EXPORT_SYMBOL(set_blocksize);
128
129int sb_set_blocksize(struct super_block *sb, int size)
130{
131 if (set_blocksize(sb->s_bdev, size))
132 return 0;
133
134
135 sb->s_blocksize = size;
136 sb->s_blocksize_bits = blksize_bits(size);
137 return sb->s_blocksize;
138}
139
140EXPORT_SYMBOL(sb_set_blocksize);
141
142int sb_min_blocksize(struct super_block *sb, int size)
143{
144 int minsize = bdev_logical_block_size(sb->s_bdev);
145 if (size < minsize)
146 size = minsize;
147 return sb_set_blocksize(sb, size);
148}
149
150EXPORT_SYMBOL(sb_min_blocksize);
151
152static int
153blkdev_get_block(struct inode *inode, sector_t iblock,
154 struct buffer_head *bh, int create)
155{
156 bh->b_bdev = I_BDEV(inode);
157 bh->b_blocknr = iblock;
158 set_buffer_mapped(bh);
159 return 0;
160}
161
162static struct inode *bdev_file_inode(struct file *file)
163{
164 return file->f_mapping->host;
165}
166
167static unsigned int dio_bio_write_op(struct kiocb *iocb)
168{
169 unsigned int op = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE;
170
171
172 if (iocb->ki_flags & IOCB_DSYNC)
173 op |= REQ_FUA;
174 return op;
175}
176
177#define DIO_INLINE_BIO_VECS 4
178
179static void blkdev_bio_end_io_simple(struct bio *bio)
180{
181 struct task_struct *waiter = bio->bi_private;
182
183 WRITE_ONCE(bio->bi_private, NULL);
184 wake_up_process(waiter);
185}
186
187static ssize_t
188__blkdev_direct_IO_simple(struct kiocb *iocb, struct iov_iter *iter,
189 int nr_pages)
190{
191 struct file *file = iocb->ki_filp;
192 struct block_device *bdev = I_BDEV(bdev_file_inode(file));
193 struct bio_vec inline_vecs[DIO_INLINE_BIO_VECS], *vecs, *bvec;
194 loff_t pos = iocb->ki_pos;
195 bool should_dirty = false;
196 struct bio bio;
197 ssize_t ret;
198 blk_qc_t qc;
199 int i;
200
201 if ((pos | iov_iter_alignment(iter)) &
202 (bdev_logical_block_size(bdev) - 1))
203 return -EINVAL;
204
205 if (nr_pages <= DIO_INLINE_BIO_VECS)
206 vecs = inline_vecs;
207 else {
208 vecs = kmalloc(nr_pages * sizeof(struct bio_vec), GFP_KERNEL);
209 if (!vecs)
210 return -ENOMEM;
211 }
212
213 bio_init(&bio, vecs, nr_pages);
214 bio_set_dev(&bio, bdev);
215 bio.bi_iter.bi_sector = pos >> 9;
216 bio.bi_write_hint = iocb->ki_hint;
217 bio.bi_private = current;
218 bio.bi_end_io = blkdev_bio_end_io_simple;
219
220 ret = bio_iov_iter_get_pages(&bio, iter);
221 if (unlikely(ret))
222 return ret;
223 ret = bio.bi_iter.bi_size;
224
225 if (iov_iter_rw(iter) == READ) {
226 bio.bi_opf = REQ_OP_READ;
227 if (iter_is_iovec(iter))
228 should_dirty = true;
229 } else {
230 bio.bi_opf = dio_bio_write_op(iocb);
231 task_io_account_write(ret);
232 }
233
234 qc = submit_bio(&bio);
235 for (;;) {
236 set_current_state(TASK_UNINTERRUPTIBLE);
237 if (!READ_ONCE(bio.bi_private))
238 break;
239 if (!(iocb->ki_flags & IOCB_HIPRI) ||
240 !blk_poll(bdev_get_queue(bdev), qc))
241 io_schedule();
242 }
243 __set_current_state(TASK_RUNNING);
244
245 bio_for_each_segment_all(bvec, &bio, i) {
246 if (should_dirty && !PageCompound(bvec->bv_page))
247 set_page_dirty_lock(bvec->bv_page);
248 put_page(bvec->bv_page);
249 }
250
251 if (vecs != inline_vecs)
252 kfree(vecs);
253
254 if (unlikely(bio.bi_status))
255 ret = blk_status_to_errno(bio.bi_status);
256
257 bio_uninit(&bio);
258
259 return ret;
260}
261
262struct blkdev_dio {
263 union {
264 struct kiocb *iocb;
265 struct task_struct *waiter;
266 };
267 size_t size;
268 atomic_t ref;
269 bool multi_bio : 1;
270 bool should_dirty : 1;
271 bool is_sync : 1;
272 struct bio bio;
273};
274
275static struct bio_set *blkdev_dio_pool __read_mostly;
276
277static void blkdev_bio_end_io(struct bio *bio)
278{
279 struct blkdev_dio *dio = bio->bi_private;
280 bool should_dirty = dio->should_dirty;
281
282 if (dio->multi_bio && !atomic_dec_and_test(&dio->ref)) {
283 if (bio->bi_status && !dio->bio.bi_status)
284 dio->bio.bi_status = bio->bi_status;
285 } else {
286 if (!dio->is_sync) {
287 struct kiocb *iocb = dio->iocb;
288 ssize_t ret;
289
290 if (likely(!dio->bio.bi_status)) {
291 ret = dio->size;
292 iocb->ki_pos += ret;
293 } else {
294 ret = blk_status_to_errno(dio->bio.bi_status);
295 }
296
297 dio->iocb->ki_complete(iocb, ret, 0);
298 bio_put(&dio->bio);
299 } else {
300 struct task_struct *waiter = dio->waiter;
301
302 WRITE_ONCE(dio->waiter, NULL);
303 wake_up_process(waiter);
304 }
305 }
306
307 if (should_dirty) {
308 bio_check_pages_dirty(bio);
309 } else {
310 struct bio_vec *bvec;
311 int i;
312
313 bio_for_each_segment_all(bvec, bio, i)
314 put_page(bvec->bv_page);
315 bio_put(bio);
316 }
317}
318
319static ssize_t
320__blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages)
321{
322 struct file *file = iocb->ki_filp;
323 struct inode *inode = bdev_file_inode(file);
324 struct block_device *bdev = I_BDEV(inode);
325 struct blk_plug plug;
326 struct blkdev_dio *dio;
327 struct bio *bio;
328 bool is_read = (iov_iter_rw(iter) == READ), is_sync;
329 loff_t pos = iocb->ki_pos;
330 blk_qc_t qc = BLK_QC_T_NONE;
331 int ret = 0;
332
333 if ((pos | iov_iter_alignment(iter)) &
334 (bdev_logical_block_size(bdev) - 1))
335 return -EINVAL;
336
337 bio = bio_alloc_bioset(GFP_KERNEL, nr_pages, blkdev_dio_pool);
338 bio_get(bio);
339
340 dio = container_of(bio, struct blkdev_dio, bio);
341 dio->is_sync = is_sync = is_sync_kiocb(iocb);
342 if (dio->is_sync)
343 dio->waiter = current;
344 else
345 dio->iocb = iocb;
346
347 dio->size = 0;
348 dio->multi_bio = false;
349 dio->should_dirty = is_read && (iter->type == ITER_IOVEC);
350
351 blk_start_plug(&plug);
352 for (;;) {
353 bio_set_dev(bio, bdev);
354 bio->bi_iter.bi_sector = pos >> 9;
355 bio->bi_write_hint = iocb->ki_hint;
356 bio->bi_private = dio;
357 bio->bi_end_io = blkdev_bio_end_io;
358
359 ret = bio_iov_iter_get_pages(bio, iter);
360 if (unlikely(ret)) {
361 bio->bi_status = BLK_STS_IOERR;
362 bio_endio(bio);
363 break;
364 }
365
366 if (is_read) {
367 bio->bi_opf = REQ_OP_READ;
368 if (dio->should_dirty)
369 bio_set_pages_dirty(bio);
370 } else {
371 bio->bi_opf = dio_bio_write_op(iocb);
372 task_io_account_write(bio->bi_iter.bi_size);
373 }
374
375 dio->size += bio->bi_iter.bi_size;
376 pos += bio->bi_iter.bi_size;
377
378 nr_pages = iov_iter_npages(iter, BIO_MAX_PAGES);
379 if (!nr_pages) {
380 qc = submit_bio(bio);
381 break;
382 }
383
384 if (!dio->multi_bio) {
385 dio->multi_bio = true;
386 atomic_set(&dio->ref, 2);
387 } else {
388 atomic_inc(&dio->ref);
389 }
390
391 submit_bio(bio);
392 bio = bio_alloc(GFP_KERNEL, nr_pages);
393 }
394 blk_finish_plug(&plug);
395
396 if (!is_sync)
397 return -EIOCBQUEUED;
398
399 for (;;) {
400 set_current_state(TASK_UNINTERRUPTIBLE);
401 if (!READ_ONCE(dio->waiter))
402 break;
403
404 if (!(iocb->ki_flags & IOCB_HIPRI) ||
405 !blk_poll(bdev_get_queue(bdev), qc))
406 io_schedule();
407 }
408 __set_current_state(TASK_RUNNING);
409
410 if (!ret)
411 ret = blk_status_to_errno(dio->bio.bi_status);
412 if (likely(!ret))
413 ret = dio->size;
414
415 bio_put(&dio->bio);
416 return ret;
417}
418
419static ssize_t
420blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
421{
422 int nr_pages;
423
424 nr_pages = iov_iter_npages(iter, BIO_MAX_PAGES + 1);
425 if (!nr_pages)
426 return 0;
427 if (is_sync_kiocb(iocb) && nr_pages <= BIO_MAX_PAGES)
428 return __blkdev_direct_IO_simple(iocb, iter, nr_pages);
429
430 return __blkdev_direct_IO(iocb, iter, min(nr_pages, BIO_MAX_PAGES));
431}
432
433static __init int blkdev_init(void)
434{
435 blkdev_dio_pool = bioset_create(4, offsetof(struct blkdev_dio, bio), BIOSET_NEED_BVECS);
436 if (!blkdev_dio_pool)
437 return -ENOMEM;
438 return 0;
439}
440module_init(blkdev_init);
441
442int __sync_blockdev(struct block_device *bdev, int wait)
443{
444 if (!bdev)
445 return 0;
446 if (!wait)
447 return filemap_flush(bdev->bd_inode->i_mapping);
448 return filemap_write_and_wait(bdev->bd_inode->i_mapping);
449}
450
451
452
453
454
455int sync_blockdev(struct block_device *bdev)
456{
457 return __sync_blockdev(bdev, 1);
458}
459EXPORT_SYMBOL(sync_blockdev);
460
461
462
463
464
465
466int fsync_bdev(struct block_device *bdev)
467{
468 struct super_block *sb = get_super(bdev);
469 if (sb) {
470 int res = sync_filesystem(sb);
471 drop_super(sb);
472 return res;
473 }
474 return sync_blockdev(bdev);
475}
476EXPORT_SYMBOL(fsync_bdev);
477
478
479
480
481
482
483
484
485
486
487
488
489
490struct super_block *freeze_bdev(struct block_device *bdev)
491{
492 struct super_block *sb;
493 int error = 0;
494
495 mutex_lock(&bdev->bd_fsfreeze_mutex);
496 if (++bdev->bd_fsfreeze_count > 1) {
497
498
499
500
501
502 sb = get_super(bdev);
503 if (sb)
504 drop_super(sb);
505 mutex_unlock(&bdev->bd_fsfreeze_mutex);
506 return sb;
507 }
508
509 sb = get_active_super(bdev);
510 if (!sb)
511 goto out;
512 if (sb->s_op->freeze_super)
513 error = sb->s_op->freeze_super(sb);
514 else
515 error = freeze_super(sb);
516 if (error) {
517 deactivate_super(sb);
518 bdev->bd_fsfreeze_count--;
519 mutex_unlock(&bdev->bd_fsfreeze_mutex);
520 return ERR_PTR(error);
521 }
522 deactivate_super(sb);
523 out:
524 sync_blockdev(bdev);
525 mutex_unlock(&bdev->bd_fsfreeze_mutex);
526 return sb;
527}
528EXPORT_SYMBOL(freeze_bdev);
529
530
531
532
533
534
535
536
537int thaw_bdev(struct block_device *bdev, struct super_block *sb)
538{
539 int error = -EINVAL;
540
541 mutex_lock(&bdev->bd_fsfreeze_mutex);
542 if (!bdev->bd_fsfreeze_count)
543 goto out;
544
545 error = 0;
546 if (--bdev->bd_fsfreeze_count > 0)
547 goto out;
548
549 if (!sb)
550 goto out;
551
552 if (sb->s_op->thaw_super)
553 error = sb->s_op->thaw_super(sb);
554 else
555 error = thaw_super(sb);
556 if (error)
557 bdev->bd_fsfreeze_count++;
558out:
559 mutex_unlock(&bdev->bd_fsfreeze_mutex);
560 return error;
561}
562EXPORT_SYMBOL(thaw_bdev);
563
564static int blkdev_writepage(struct page *page, struct writeback_control *wbc)
565{
566 return block_write_full_page(page, blkdev_get_block, wbc);
567}
568
569static int blkdev_readpage(struct file * file, struct page * page)
570{
571 return block_read_full_page(page, blkdev_get_block);
572}
573
574static int blkdev_readpages(struct file *file, struct address_space *mapping,
575 struct list_head *pages, unsigned nr_pages)
576{
577 return mpage_readpages(mapping, pages, nr_pages, blkdev_get_block);
578}
579
580static int blkdev_write_begin(struct file *file, struct address_space *mapping,
581 loff_t pos, unsigned len, unsigned flags,
582 struct page **pagep, void **fsdata)
583{
584 return block_write_begin(mapping, pos, len, flags, pagep,
585 blkdev_get_block);
586}
587
588static int blkdev_write_end(struct file *file, struct address_space *mapping,
589 loff_t pos, unsigned len, unsigned copied,
590 struct page *page, void *fsdata)
591{
592 int ret;
593 ret = block_write_end(file, mapping, pos, len, copied, page, fsdata);
594
595 unlock_page(page);
596 put_page(page);
597
598 return ret;
599}
600
601
602
603
604
605
606static loff_t block_llseek(struct file *file, loff_t offset, int whence)
607{
608 struct inode *bd_inode = bdev_file_inode(file);
609 loff_t retval;
610
611 inode_lock(bd_inode);
612 retval = fixed_size_llseek(file, offset, whence, i_size_read(bd_inode));
613 inode_unlock(bd_inode);
614 return retval;
615}
616
617int blkdev_fsync(struct file *filp, loff_t start, loff_t end, int datasync)
618{
619 struct inode *bd_inode = bdev_file_inode(filp);
620 struct block_device *bdev = I_BDEV(bd_inode);
621 int error;
622
623 error = file_write_and_wait_range(filp, start, end);
624 if (error)
625 return error;
626
627
628
629
630
631
632 error = blkdev_issue_flush(bdev, GFP_KERNEL, NULL);
633 if (error == -EOPNOTSUPP)
634 error = 0;
635
636 return error;
637}
638EXPORT_SYMBOL(blkdev_fsync);
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656int bdev_read_page(struct block_device *bdev, sector_t sector,
657 struct page *page)
658{
659 const struct block_device_operations *ops = bdev->bd_disk->fops;
660 int result = -EOPNOTSUPP;
661
662 if (!ops->rw_page || bdev_get_integrity(bdev))
663 return result;
664
665 result = blk_queue_enter(bdev->bd_queue, 0);
666 if (result)
667 return result;
668 result = ops->rw_page(bdev, sector + get_start_sect(bdev), page, false);
669 blk_queue_exit(bdev->bd_queue);
670 return result;
671}
672EXPORT_SYMBOL_GPL(bdev_read_page);
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693int bdev_write_page(struct block_device *bdev, sector_t sector,
694 struct page *page, struct writeback_control *wbc)
695{
696 int result;
697 const struct block_device_operations *ops = bdev->bd_disk->fops;
698
699 if (!ops->rw_page || bdev_get_integrity(bdev))
700 return -EOPNOTSUPP;
701 result = blk_queue_enter(bdev->bd_queue, 0);
702 if (result)
703 return result;
704
705 set_page_writeback(page);
706 result = ops->rw_page(bdev, sector + get_start_sect(bdev), page, true);
707 if (result) {
708 end_page_writeback(page);
709 } else {
710 clean_page_buffers(page);
711 unlock_page(page);
712 }
713 blk_queue_exit(bdev->bd_queue);
714 return result;
715}
716EXPORT_SYMBOL_GPL(bdev_write_page);
717
718
719
720
721
722static __cacheline_aligned_in_smp DEFINE_SPINLOCK(bdev_lock);
723static struct kmem_cache * bdev_cachep __read_mostly;
724
725static struct inode *bdev_alloc_inode(struct super_block *sb)
726{
727 struct bdev_inode *ei = kmem_cache_alloc(bdev_cachep, GFP_KERNEL);
728 if (!ei)
729 return NULL;
730 return &ei->vfs_inode;
731}
732
733static void bdev_i_callback(struct rcu_head *head)
734{
735 struct inode *inode = container_of(head, struct inode, i_rcu);
736 struct bdev_inode *bdi = BDEV_I(inode);
737
738 kmem_cache_free(bdev_cachep, bdi);
739}
740
741static void bdev_destroy_inode(struct inode *inode)
742{
743 call_rcu(&inode->i_rcu, bdev_i_callback);
744}
745
746static void init_once(void *foo)
747{
748 struct bdev_inode *ei = (struct bdev_inode *) foo;
749 struct block_device *bdev = &ei->bdev;
750
751 memset(bdev, 0, sizeof(*bdev));
752 mutex_init(&bdev->bd_mutex);
753 INIT_LIST_HEAD(&bdev->bd_list);
754#ifdef CONFIG_SYSFS
755 INIT_LIST_HEAD(&bdev->bd_holder_disks);
756#endif
757 bdev->bd_bdi = &noop_backing_dev_info;
758 inode_init_once(&ei->vfs_inode);
759
760 mutex_init(&bdev->bd_fsfreeze_mutex);
761}
762
763static void bdev_evict_inode(struct inode *inode)
764{
765 struct block_device *bdev = &BDEV_I(inode)->bdev;
766 truncate_inode_pages_final(&inode->i_data);
767 invalidate_inode_buffers(inode);
768 clear_inode(inode);
769 spin_lock(&bdev_lock);
770 list_del_init(&bdev->bd_list);
771 spin_unlock(&bdev_lock);
772
773 inode_detach_wb(inode);
774 if (bdev->bd_bdi != &noop_backing_dev_info) {
775 bdi_put(bdev->bd_bdi);
776 bdev->bd_bdi = &noop_backing_dev_info;
777 }
778}
779
780static const struct super_operations bdev_sops = {
781 .statfs = simple_statfs,
782 .alloc_inode = bdev_alloc_inode,
783 .destroy_inode = bdev_destroy_inode,
784 .drop_inode = generic_delete_inode,
785 .evict_inode = bdev_evict_inode,
786};
787
788static struct dentry *bd_mount(struct file_system_type *fs_type,
789 int flags, const char *dev_name, void *data)
790{
791 struct dentry *dent;
792 dent = mount_pseudo(fs_type, "bdev:", &bdev_sops, NULL, BDEVFS_MAGIC);
793 if (!IS_ERR(dent))
794 dent->d_sb->s_iflags |= SB_I_CGROUPWB;
795 return dent;
796}
797
798static struct file_system_type bd_type = {
799 .name = "bdev",
800 .mount = bd_mount,
801 .kill_sb = kill_anon_super,
802};
803
804struct super_block *blockdev_superblock __read_mostly;
805EXPORT_SYMBOL_GPL(blockdev_superblock);
806
807void __init bdev_cache_init(void)
808{
809 int err;
810 static struct vfsmount *bd_mnt;
811
812 bdev_cachep = kmem_cache_create("bdev_cache", sizeof(struct bdev_inode),
813 0, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT|
814 SLAB_MEM_SPREAD|SLAB_ACCOUNT|SLAB_PANIC),
815 init_once);
816 err = register_filesystem(&bd_type);
817 if (err)
818 panic("Cannot register bdev pseudo-fs");
819 bd_mnt = kern_mount(&bd_type);
820 if (IS_ERR(bd_mnt))
821 panic("Cannot create bdev pseudo-fs");
822 blockdev_superblock = bd_mnt->mnt_sb;
823}
824
825
826
827
828
829
830static inline unsigned long hash(dev_t dev)
831{
832 return MAJOR(dev)+MINOR(dev);
833}
834
835static int bdev_test(struct inode *inode, void *data)
836{
837 return BDEV_I(inode)->bdev.bd_dev == *(dev_t *)data;
838}
839
840static int bdev_set(struct inode *inode, void *data)
841{
842 BDEV_I(inode)->bdev.bd_dev = *(dev_t *)data;
843 return 0;
844}
845
846static LIST_HEAD(all_bdevs);
847
848
849
850
851
852void bdev_unhash_inode(dev_t dev)
853{
854 struct inode *inode;
855
856 inode = ilookup5(blockdev_superblock, hash(dev), bdev_test, &dev);
857 if (inode) {
858 remove_inode_hash(inode);
859 iput(inode);
860 }
861}
862
863struct block_device *bdget(dev_t dev)
864{
865 struct block_device *bdev;
866 struct inode *inode;
867
868 inode = iget5_locked(blockdev_superblock, hash(dev),
869 bdev_test, bdev_set, &dev);
870
871 if (!inode)
872 return NULL;
873
874 bdev = &BDEV_I(inode)->bdev;
875
876 if (inode->i_state & I_NEW) {
877 bdev->bd_contains = NULL;
878 bdev->bd_super = NULL;
879 bdev->bd_inode = inode;
880 bdev->bd_block_size = i_blocksize(inode);
881 bdev->bd_part_count = 0;
882 bdev->bd_invalidated = 0;
883 inode->i_mode = S_IFBLK;
884 inode->i_rdev = dev;
885 inode->i_bdev = bdev;
886 inode->i_data.a_ops = &def_blk_aops;
887 mapping_set_gfp_mask(&inode->i_data, GFP_USER);
888 spin_lock(&bdev_lock);
889 list_add(&bdev->bd_list, &all_bdevs);
890 spin_unlock(&bdev_lock);
891 unlock_new_inode(inode);
892 }
893 return bdev;
894}
895
896EXPORT_SYMBOL(bdget);
897
898
899
900
901
902struct block_device *bdgrab(struct block_device *bdev)
903{
904 ihold(bdev->bd_inode);
905 return bdev;
906}
907EXPORT_SYMBOL(bdgrab);
908
909long nr_blockdev_pages(void)
910{
911 struct block_device *bdev;
912 long ret = 0;
913 spin_lock(&bdev_lock);
914 list_for_each_entry(bdev, &all_bdevs, bd_list) {
915 ret += bdev->bd_inode->i_mapping->nrpages;
916 }
917 spin_unlock(&bdev_lock);
918 return ret;
919}
920
921void bdput(struct block_device *bdev)
922{
923 iput(bdev->bd_inode);
924}
925
926EXPORT_SYMBOL(bdput);
927
928static struct block_device *bd_acquire(struct inode *inode)
929{
930 struct block_device *bdev;
931
932 spin_lock(&bdev_lock);
933 bdev = inode->i_bdev;
934 if (bdev && !inode_unhashed(bdev->bd_inode)) {
935 bdgrab(bdev);
936 spin_unlock(&bdev_lock);
937 return bdev;
938 }
939 spin_unlock(&bdev_lock);
940
941
942
943
944
945
946
947 if (bdev)
948 bd_forget(inode);
949
950 bdev = bdget(inode->i_rdev);
951 if (bdev) {
952 spin_lock(&bdev_lock);
953 if (!inode->i_bdev) {
954
955
956
957
958
959
960 bdgrab(bdev);
961 inode->i_bdev = bdev;
962 inode->i_mapping = bdev->bd_inode->i_mapping;
963 }
964 spin_unlock(&bdev_lock);
965 }
966 return bdev;
967}
968
969
970
971void bd_forget(struct inode *inode)
972{
973 struct block_device *bdev = NULL;
974
975 spin_lock(&bdev_lock);
976 if (!sb_is_blkdev_sb(inode->i_sb))
977 bdev = inode->i_bdev;
978 inode->i_bdev = NULL;
979 inode->i_mapping = &inode->i_data;
980 spin_unlock(&bdev_lock);
981
982 if (bdev)
983 bdput(bdev);
984}
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000static bool bd_may_claim(struct block_device *bdev, struct block_device *whole,
1001 void *holder)
1002{
1003 if (bdev->bd_holder == holder)
1004 return true;
1005 else if (bdev->bd_holder != NULL)
1006 return false;
1007 else if (whole == bdev)
1008 return true;
1009
1010 else if (whole->bd_holder == bd_may_claim)
1011 return true;
1012 else if (whole->bd_holder != NULL)
1013 return false;
1014 else
1015 return true;
1016}
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036static int bd_prepare_to_claim(struct block_device *bdev,
1037 struct block_device *whole, void *holder)
1038{
1039retry:
1040
1041 if (!bd_may_claim(bdev, whole, holder))
1042 return -EBUSY;
1043
1044
1045 if (whole->bd_claiming) {
1046 wait_queue_head_t *wq = bit_waitqueue(&whole->bd_claiming, 0);
1047 DEFINE_WAIT(wait);
1048
1049 prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE);
1050 spin_unlock(&bdev_lock);
1051 schedule();
1052 finish_wait(wq, &wait);
1053 spin_lock(&bdev_lock);
1054 goto retry;
1055 }
1056
1057
1058 return 0;
1059}
1060
1061static struct gendisk *bdev_get_gendisk(struct block_device *bdev, int *partno)
1062{
1063 struct gendisk *disk = get_gendisk(bdev->bd_dev, partno);
1064
1065 if (!disk)
1066 return NULL;
1067
1068
1069
1070
1071
1072
1073
1074
1075 if (inode_unhashed(bdev->bd_inode)) {
1076 put_disk_and_module(disk);
1077 return NULL;
1078 }
1079 return disk;
1080}
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105static struct block_device *bd_start_claiming(struct block_device *bdev,
1106 void *holder)
1107{
1108 struct gendisk *disk;
1109 struct block_device *whole;
1110 int partno, err;
1111
1112 might_sleep();
1113
1114
1115
1116
1117
1118 disk = bdev_get_gendisk(bdev, &partno);
1119 if (!disk)
1120 return ERR_PTR(-ENXIO);
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130 if (partno)
1131 whole = bdget_disk(disk, 0);
1132 else
1133 whole = bdgrab(bdev);
1134
1135 put_disk_and_module(disk);
1136 if (!whole)
1137 return ERR_PTR(-ENOMEM);
1138
1139
1140 spin_lock(&bdev_lock);
1141
1142 err = bd_prepare_to_claim(bdev, whole, holder);
1143 if (err == 0) {
1144 whole->bd_claiming = holder;
1145 spin_unlock(&bdev_lock);
1146 return whole;
1147 } else {
1148 spin_unlock(&bdev_lock);
1149 bdput(whole);
1150 return ERR_PTR(err);
1151 }
1152}
1153
1154#ifdef CONFIG_SYSFS
1155struct bd_holder_disk {
1156 struct list_head list;
1157 struct gendisk *disk;
1158 int refcnt;
1159};
1160
1161static struct bd_holder_disk *bd_find_holder_disk(struct block_device *bdev,
1162 struct gendisk *disk)
1163{
1164 struct bd_holder_disk *holder;
1165
1166 list_for_each_entry(holder, &bdev->bd_holder_disks, list)
1167 if (holder->disk == disk)
1168 return holder;
1169 return NULL;
1170}
1171
1172static int add_symlink(struct kobject *from, struct kobject *to)
1173{
1174 return sysfs_create_link(from, to, kobject_name(to));
1175}
1176
1177static void del_symlink(struct kobject *from, struct kobject *to)
1178{
1179 sysfs_remove_link(from, kobject_name(to));
1180}
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210int bd_link_disk_holder(struct block_device *bdev, struct gendisk *disk)
1211{
1212 struct bd_holder_disk *holder;
1213 int ret = 0;
1214
1215 mutex_lock(&bdev->bd_mutex);
1216
1217 WARN_ON_ONCE(!bdev->bd_holder);
1218
1219
1220 if (WARN_ON(!disk->slave_dir || !bdev->bd_part->holder_dir))
1221 goto out_unlock;
1222
1223 holder = bd_find_holder_disk(bdev, disk);
1224 if (holder) {
1225 holder->refcnt++;
1226 goto out_unlock;
1227 }
1228
1229 holder = kzalloc(sizeof(*holder), GFP_KERNEL);
1230 if (!holder) {
1231 ret = -ENOMEM;
1232 goto out_unlock;
1233 }
1234
1235 INIT_LIST_HEAD(&holder->list);
1236 holder->disk = disk;
1237 holder->refcnt = 1;
1238
1239 ret = add_symlink(disk->slave_dir, &part_to_dev(bdev->bd_part)->kobj);
1240 if (ret)
1241 goto out_free;
1242
1243 ret = add_symlink(bdev->bd_part->holder_dir, &disk_to_dev(disk)->kobj);
1244 if (ret)
1245 goto out_del;
1246
1247
1248
1249
1250 kobject_get(bdev->bd_part->holder_dir);
1251
1252 list_add(&holder->list, &bdev->bd_holder_disks);
1253 goto out_unlock;
1254
1255out_del:
1256 del_symlink(disk->slave_dir, &part_to_dev(bdev->bd_part)->kobj);
1257out_free:
1258 kfree(holder);
1259out_unlock:
1260 mutex_unlock(&bdev->bd_mutex);
1261 return ret;
1262}
1263EXPORT_SYMBOL_GPL(bd_link_disk_holder);
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275void bd_unlink_disk_holder(struct block_device *bdev, struct gendisk *disk)
1276{
1277 struct bd_holder_disk *holder;
1278
1279 mutex_lock(&bdev->bd_mutex);
1280
1281 holder = bd_find_holder_disk(bdev, disk);
1282
1283 if (!WARN_ON_ONCE(holder == NULL) && !--holder->refcnt) {
1284 del_symlink(disk->slave_dir, &part_to_dev(bdev->bd_part)->kobj);
1285 del_symlink(bdev->bd_part->holder_dir,
1286 &disk_to_dev(disk)->kobj);
1287 kobject_put(bdev->bd_part->holder_dir);
1288 list_del_init(&holder->list);
1289 kfree(holder);
1290 }
1291
1292 mutex_unlock(&bdev->bd_mutex);
1293}
1294EXPORT_SYMBOL_GPL(bd_unlink_disk_holder);
1295#endif
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307static void flush_disk(struct block_device *bdev, bool kill_dirty)
1308{
1309 if (__invalidate_device(bdev, kill_dirty)) {
1310 printk(KERN_WARNING "VFS: busy inodes on changed media or "
1311 "resized disk %s\n",
1312 bdev->bd_disk ? bdev->bd_disk->disk_name : "");
1313 }
1314
1315 if (!bdev->bd_disk)
1316 return;
1317 if (disk_part_scan_enabled(bdev->bd_disk))
1318 bdev->bd_invalidated = 1;
1319}
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329void check_disk_size_change(struct gendisk *disk, struct block_device *bdev)
1330{
1331 loff_t disk_size, bdev_size;
1332
1333 disk_size = (loff_t)get_capacity(disk) << 9;
1334 bdev_size = i_size_read(bdev->bd_inode);
1335 if (disk_size != bdev_size) {
1336 printk(KERN_INFO
1337 "%s: detected capacity change from %lld to %lld\n",
1338 disk->disk_name, bdev_size, disk_size);
1339 i_size_write(bdev->bd_inode, disk_size);
1340 flush_disk(bdev, false);
1341 }
1342}
1343EXPORT_SYMBOL(check_disk_size_change);
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353int revalidate_disk(struct gendisk *disk)
1354{
1355 struct block_device *bdev;
1356 int ret = 0;
1357
1358 if (disk->fops->revalidate_disk)
1359 ret = disk->fops->revalidate_disk(disk);
1360 bdev = bdget_disk(disk, 0);
1361 if (!bdev)
1362 return ret;
1363
1364 mutex_lock(&bdev->bd_mutex);
1365 check_disk_size_change(disk, bdev);
1366 bdev->bd_invalidated = 0;
1367 mutex_unlock(&bdev->bd_mutex);
1368 bdput(bdev);
1369 return ret;
1370}
1371EXPORT_SYMBOL(revalidate_disk);
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382int check_disk_change(struct block_device *bdev)
1383{
1384 struct gendisk *disk = bdev->bd_disk;
1385 const struct block_device_operations *bdops = disk->fops;
1386 unsigned int events;
1387
1388 events = disk_clear_events(disk, DISK_EVENT_MEDIA_CHANGE |
1389 DISK_EVENT_EJECT_REQUEST);
1390 if (!(events & DISK_EVENT_MEDIA_CHANGE))
1391 return 0;
1392
1393 flush_disk(bdev, true);
1394 if (bdops->revalidate_disk)
1395 bdops->revalidate_disk(bdev->bd_disk);
1396 return 1;
1397}
1398
1399EXPORT_SYMBOL(check_disk_change);
1400
1401void bd_set_size(struct block_device *bdev, loff_t size)
1402{
1403 unsigned bsize = bdev_logical_block_size(bdev);
1404
1405 inode_lock(bdev->bd_inode);
1406 i_size_write(bdev->bd_inode, size);
1407 inode_unlock(bdev->bd_inode);
1408 while (bsize < PAGE_SIZE) {
1409 if (size & bsize)
1410 break;
1411 bsize <<= 1;
1412 }
1413 bdev->bd_block_size = bsize;
1414 bdev->bd_inode->i_blkbits = blksize_bits(bsize);
1415}
1416EXPORT_SYMBOL(bd_set_size);
1417
1418static void __blkdev_put(struct block_device *bdev, fmode_t mode, int for_part);
1419
1420
1421
1422
1423
1424
1425
1426
1427static int __blkdev_get(struct block_device *bdev, fmode_t mode, int for_part)
1428{
1429 struct gendisk *disk;
1430 int ret;
1431 int partno;
1432 int perm = 0;
1433 bool first_open = false;
1434
1435 if (mode & FMODE_READ)
1436 perm |= MAY_READ;
1437 if (mode & FMODE_WRITE)
1438 perm |= MAY_WRITE;
1439
1440
1441
1442 if (!for_part) {
1443 ret = devcgroup_inode_permission(bdev->bd_inode, perm);
1444 if (ret != 0) {
1445 bdput(bdev);
1446 return ret;
1447 }
1448 }
1449
1450 restart:
1451
1452 ret = -ENXIO;
1453 disk = bdev_get_gendisk(bdev, &partno);
1454 if (!disk)
1455 goto out;
1456
1457 disk_block_events(disk);
1458 mutex_lock_nested(&bdev->bd_mutex, for_part);
1459 if (!bdev->bd_openers) {
1460 first_open = true;
1461 bdev->bd_disk = disk;
1462 bdev->bd_queue = disk->queue;
1463 bdev->bd_contains = bdev;
1464 bdev->bd_partno = partno;
1465
1466 if (!partno) {
1467 ret = -ENXIO;
1468 bdev->bd_part = disk_get_part(disk, partno);
1469 if (!bdev->bd_part)
1470 goto out_clear;
1471
1472 ret = 0;
1473 if (disk->fops->open) {
1474 ret = disk->fops->open(bdev, mode);
1475 if (ret == -ERESTARTSYS) {
1476
1477
1478
1479
1480 disk_put_part(bdev->bd_part);
1481 bdev->bd_part = NULL;
1482 bdev->bd_disk = NULL;
1483 bdev->bd_queue = NULL;
1484 mutex_unlock(&bdev->bd_mutex);
1485 disk_unblock_events(disk);
1486 put_disk_and_module(disk);
1487 goto restart;
1488 }
1489 }
1490
1491 if (!ret)
1492 bd_set_size(bdev,(loff_t)get_capacity(disk)<<9);
1493
1494
1495
1496
1497
1498
1499
1500 if (bdev->bd_invalidated) {
1501 if (!ret)
1502 rescan_partitions(disk, bdev);
1503 else if (ret == -ENOMEDIUM)
1504 invalidate_partitions(disk, bdev);
1505 }
1506
1507 if (ret)
1508 goto out_clear;
1509 } else {
1510 struct block_device *whole;
1511 whole = bdget_disk(disk, 0);
1512 ret = -ENOMEM;
1513 if (!whole)
1514 goto out_clear;
1515 BUG_ON(for_part);
1516 ret = __blkdev_get(whole, mode, 1);
1517 if (ret)
1518 goto out_clear;
1519 bdev->bd_contains = whole;
1520 bdev->bd_part = disk_get_part(disk, partno);
1521 if (!(disk->flags & GENHD_FL_UP) ||
1522 !bdev->bd_part || !bdev->bd_part->nr_sects) {
1523 ret = -ENXIO;
1524 goto out_clear;
1525 }
1526 bd_set_size(bdev, (loff_t)bdev->bd_part->nr_sects << 9);
1527 }
1528
1529 if (bdev->bd_bdi == &noop_backing_dev_info)
1530 bdev->bd_bdi = bdi_get(disk->queue->backing_dev_info);
1531 } else {
1532 if (bdev->bd_contains == bdev) {
1533 ret = 0;
1534 if (bdev->bd_disk->fops->open)
1535 ret = bdev->bd_disk->fops->open(bdev, mode);
1536
1537 if (bdev->bd_invalidated) {
1538 if (!ret)
1539 rescan_partitions(bdev->bd_disk, bdev);
1540 else if (ret == -ENOMEDIUM)
1541 invalidate_partitions(bdev->bd_disk, bdev);
1542 }
1543 if (ret)
1544 goto out_unlock_bdev;
1545 }
1546 }
1547 bdev->bd_openers++;
1548 if (for_part)
1549 bdev->bd_part_count++;
1550 mutex_unlock(&bdev->bd_mutex);
1551 disk_unblock_events(disk);
1552
1553 if (!first_open)
1554 put_disk_and_module(disk);
1555 return 0;
1556
1557 out_clear:
1558 disk_put_part(bdev->bd_part);
1559 bdev->bd_disk = NULL;
1560 bdev->bd_part = NULL;
1561 bdev->bd_queue = NULL;
1562 if (bdev != bdev->bd_contains)
1563 __blkdev_put(bdev->bd_contains, mode, 1);
1564 bdev->bd_contains = NULL;
1565 out_unlock_bdev:
1566 mutex_unlock(&bdev->bd_mutex);
1567 disk_unblock_events(disk);
1568 put_disk_and_module(disk);
1569 out:
1570 bdput(bdev);
1571
1572 return ret;
1573}
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594int blkdev_get(struct block_device *bdev, fmode_t mode, void *holder)
1595{
1596 struct block_device *whole = NULL;
1597 int res;
1598
1599 WARN_ON_ONCE((mode & FMODE_EXCL) && !holder);
1600
1601 if ((mode & FMODE_EXCL) && holder) {
1602 whole = bd_start_claiming(bdev, holder);
1603 if (IS_ERR(whole)) {
1604 bdput(bdev);
1605 return PTR_ERR(whole);
1606 }
1607 }
1608
1609 res = __blkdev_get(bdev, mode, 0);
1610
1611 if (whole) {
1612 struct gendisk *disk = whole->bd_disk;
1613
1614
1615 mutex_lock(&bdev->bd_mutex);
1616 spin_lock(&bdev_lock);
1617
1618 if (!res) {
1619 BUG_ON(!bd_may_claim(bdev, whole, holder));
1620
1621
1622
1623
1624
1625
1626 whole->bd_holders++;
1627 whole->bd_holder = bd_may_claim;
1628 bdev->bd_holders++;
1629 bdev->bd_holder = holder;
1630 }
1631
1632
1633 BUG_ON(whole->bd_claiming != holder);
1634 whole->bd_claiming = NULL;
1635 wake_up_bit(&whole->bd_claiming, 0);
1636
1637 spin_unlock(&bdev_lock);
1638
1639
1640
1641
1642
1643
1644
1645
1646 if (!res && (mode & FMODE_WRITE) && !bdev->bd_write_holder &&
1647 (disk->flags & GENHD_FL_BLOCK_EVENTS_ON_EXCL_WRITE)) {
1648 bdev->bd_write_holder = true;
1649 disk_block_events(disk);
1650 }
1651
1652 mutex_unlock(&bdev->bd_mutex);
1653 bdput(whole);
1654 }
1655
1656 return res;
1657}
1658EXPORT_SYMBOL(blkdev_get);
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677struct block_device *blkdev_get_by_path(const char *path, fmode_t mode,
1678 void *holder)
1679{
1680 struct block_device *bdev;
1681 int err;
1682
1683 bdev = lookup_bdev(path);
1684 if (IS_ERR(bdev))
1685 return bdev;
1686
1687 err = blkdev_get(bdev, mode, holder);
1688 if (err)
1689 return ERR_PTR(err);
1690
1691 if ((mode & FMODE_WRITE) && bdev_read_only(bdev)) {
1692 blkdev_put(bdev, mode);
1693 return ERR_PTR(-EACCES);
1694 }
1695
1696 return bdev;
1697}
1698EXPORT_SYMBOL(blkdev_get_by_path);
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722struct block_device *blkdev_get_by_dev(dev_t dev, fmode_t mode, void *holder)
1723{
1724 struct block_device *bdev;
1725 int err;
1726
1727 bdev = bdget(dev);
1728 if (!bdev)
1729 return ERR_PTR(-ENOMEM);
1730
1731 err = blkdev_get(bdev, mode, holder);
1732 if (err)
1733 return ERR_PTR(err);
1734
1735 return bdev;
1736}
1737EXPORT_SYMBOL(blkdev_get_by_dev);
1738
1739static int blkdev_open(struct inode * inode, struct file * filp)
1740{
1741 struct block_device *bdev;
1742
1743
1744
1745
1746
1747
1748
1749 filp->f_flags |= O_LARGEFILE;
1750
1751 filp->f_mode |= FMODE_NOWAIT;
1752
1753 if (filp->f_flags & O_NDELAY)
1754 filp->f_mode |= FMODE_NDELAY;
1755 if (filp->f_flags & O_EXCL)
1756 filp->f_mode |= FMODE_EXCL;
1757 if ((filp->f_flags & O_ACCMODE) == 3)
1758 filp->f_mode |= FMODE_WRITE_IOCTL;
1759
1760 bdev = bd_acquire(inode);
1761 if (bdev == NULL)
1762 return -ENOMEM;
1763
1764 filp->f_mapping = bdev->bd_inode->i_mapping;
1765 filp->f_wb_err = filemap_sample_wb_err(filp->f_mapping);
1766
1767 return blkdev_get(bdev, filp->f_mode, filp);
1768}
1769
1770static void __blkdev_put(struct block_device *bdev, fmode_t mode, int for_part)
1771{
1772 struct gendisk *disk = bdev->bd_disk;
1773 struct block_device *victim = NULL;
1774
1775 mutex_lock_nested(&bdev->bd_mutex, for_part);
1776 if (for_part)
1777 bdev->bd_part_count--;
1778
1779 if (!--bdev->bd_openers) {
1780 WARN_ON_ONCE(bdev->bd_holders);
1781 sync_blockdev(bdev);
1782 kill_bdev(bdev);
1783
1784 bdev_write_inode(bdev);
1785 }
1786 if (bdev->bd_contains == bdev) {
1787 if (disk->fops->release)
1788 disk->fops->release(disk, mode);
1789 }
1790 if (!bdev->bd_openers) {
1791 disk_put_part(bdev->bd_part);
1792 bdev->bd_part = NULL;
1793 bdev->bd_disk = NULL;
1794 if (bdev != bdev->bd_contains)
1795 victim = bdev->bd_contains;
1796 bdev->bd_contains = NULL;
1797
1798 put_disk_and_module(disk);
1799 }
1800 mutex_unlock(&bdev->bd_mutex);
1801 bdput(bdev);
1802 if (victim)
1803 __blkdev_put(victim, mode, 1);
1804}
1805
1806void blkdev_put(struct block_device *bdev, fmode_t mode)
1807{
1808 mutex_lock(&bdev->bd_mutex);
1809
1810 if (mode & FMODE_EXCL) {
1811 bool bdev_free;
1812
1813
1814
1815
1816
1817
1818 spin_lock(&bdev_lock);
1819
1820 WARN_ON_ONCE(--bdev->bd_holders < 0);
1821 WARN_ON_ONCE(--bdev->bd_contains->bd_holders < 0);
1822
1823
1824 if ((bdev_free = !bdev->bd_holders))
1825 bdev->bd_holder = NULL;
1826 if (!bdev->bd_contains->bd_holders)
1827 bdev->bd_contains->bd_holder = NULL;
1828
1829 spin_unlock(&bdev_lock);
1830
1831
1832
1833
1834
1835 if (bdev_free && bdev->bd_write_holder) {
1836 disk_unblock_events(bdev->bd_disk);
1837 bdev->bd_write_holder = false;
1838 }
1839 }
1840
1841
1842
1843
1844
1845
1846 disk_flush_events(bdev->bd_disk, DISK_EVENT_MEDIA_CHANGE);
1847
1848 mutex_unlock(&bdev->bd_mutex);
1849
1850 __blkdev_put(bdev, mode, 0);
1851}
1852EXPORT_SYMBOL(blkdev_put);
1853
1854static int blkdev_close(struct inode * inode, struct file * filp)
1855{
1856 struct block_device *bdev = I_BDEV(bdev_file_inode(filp));
1857 blkdev_put(bdev, filp->f_mode);
1858 return 0;
1859}
1860
1861static long block_ioctl(struct file *file, unsigned cmd, unsigned long arg)
1862{
1863 struct block_device *bdev = I_BDEV(bdev_file_inode(file));
1864 fmode_t mode = file->f_mode;
1865
1866
1867
1868
1869
1870 if (file->f_flags & O_NDELAY)
1871 mode |= FMODE_NDELAY;
1872 else
1873 mode &= ~FMODE_NDELAY;
1874
1875 return blkdev_ioctl(bdev, mode, cmd, arg);
1876}
1877
1878
1879
1880
1881
1882
1883
1884
1885ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from)
1886{
1887 struct file *file = iocb->ki_filp;
1888 struct inode *bd_inode = bdev_file_inode(file);
1889 loff_t size = i_size_read(bd_inode);
1890 struct blk_plug plug;
1891 ssize_t ret;
1892
1893 if (bdev_read_only(I_BDEV(bd_inode)))
1894 return -EPERM;
1895
1896 if (!iov_iter_count(from))
1897 return 0;
1898
1899 if (iocb->ki_pos >= size)
1900 return -ENOSPC;
1901
1902 if ((iocb->ki_flags & (IOCB_NOWAIT | IOCB_DIRECT)) == IOCB_NOWAIT)
1903 return -EOPNOTSUPP;
1904
1905 iov_iter_truncate(from, size - iocb->ki_pos);
1906
1907 blk_start_plug(&plug);
1908 ret = __generic_file_write_iter(iocb, from);
1909 if (ret > 0)
1910 ret = generic_write_sync(iocb, ret);
1911 blk_finish_plug(&plug);
1912 return ret;
1913}
1914EXPORT_SYMBOL_GPL(blkdev_write_iter);
1915
1916ssize_t blkdev_read_iter(struct kiocb *iocb, struct iov_iter *to)
1917{
1918 struct file *file = iocb->ki_filp;
1919 struct inode *bd_inode = bdev_file_inode(file);
1920 loff_t size = i_size_read(bd_inode);
1921 loff_t pos = iocb->ki_pos;
1922
1923 if (pos >= size)
1924 return 0;
1925
1926 size -= pos;
1927 iov_iter_truncate(to, size);
1928 return generic_file_read_iter(iocb, to);
1929}
1930EXPORT_SYMBOL_GPL(blkdev_read_iter);
1931
1932
1933
1934
1935
1936static int blkdev_releasepage(struct page *page, gfp_t wait)
1937{
1938 struct super_block *super = BDEV_I(page->mapping->host)->bdev.bd_super;
1939
1940 if (super && super->s_op->bdev_try_to_free_page)
1941 return super->s_op->bdev_try_to_free_page(super, page, wait);
1942
1943 return try_to_free_buffers(page);
1944}
1945
1946static int blkdev_writepages(struct address_space *mapping,
1947 struct writeback_control *wbc)
1948{
1949 if (dax_mapping(mapping)) {
1950 struct block_device *bdev = I_BDEV(mapping->host);
1951
1952 return dax_writeback_mapping_range(mapping, bdev, wbc);
1953 }
1954 return generic_writepages(mapping, wbc);
1955}
1956
1957static const struct address_space_operations def_blk_aops = {
1958 .readpage = blkdev_readpage,
1959 .readpages = blkdev_readpages,
1960 .writepage = blkdev_writepage,
1961 .write_begin = blkdev_write_begin,
1962 .write_end = blkdev_write_end,
1963 .writepages = blkdev_writepages,
1964 .releasepage = blkdev_releasepage,
1965 .direct_IO = blkdev_direct_IO,
1966 .is_dirty_writeback = buffer_check_dirty_writeback,
1967};
1968
1969#define BLKDEV_FALLOC_FL_SUPPORTED \
1970 (FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE | \
1971 FALLOC_FL_ZERO_RANGE | FALLOC_FL_NO_HIDE_STALE)
1972
1973static long blkdev_fallocate(struct file *file, int mode, loff_t start,
1974 loff_t len)
1975{
1976 struct block_device *bdev = I_BDEV(bdev_file_inode(file));
1977 struct address_space *mapping;
1978 loff_t end = start + len - 1;
1979 loff_t isize;
1980 int error;
1981
1982
1983 if (mode & ~BLKDEV_FALLOC_FL_SUPPORTED)
1984 return -EOPNOTSUPP;
1985
1986
1987 isize = i_size_read(bdev->bd_inode);
1988 if (start >= isize)
1989 return -EINVAL;
1990 if (end >= isize) {
1991 if (mode & FALLOC_FL_KEEP_SIZE) {
1992 len = isize - start;
1993 end = start + len - 1;
1994 } else
1995 return -EINVAL;
1996 }
1997
1998
1999
2000
2001 if ((start | len) & (bdev_logical_block_size(bdev) - 1))
2002 return -EINVAL;
2003
2004
2005 mapping = bdev->bd_inode->i_mapping;
2006 truncate_inode_pages_range(mapping, start, end);
2007
2008 switch (mode) {
2009 case FALLOC_FL_ZERO_RANGE:
2010 case FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE:
2011 error = blkdev_issue_zeroout(bdev, start >> 9, len >> 9,
2012 GFP_KERNEL, BLKDEV_ZERO_NOUNMAP);
2013 break;
2014 case FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE:
2015 error = blkdev_issue_zeroout(bdev, start >> 9, len >> 9,
2016 GFP_KERNEL, BLKDEV_ZERO_NOFALLBACK);
2017 break;
2018 case FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE | FALLOC_FL_NO_HIDE_STALE:
2019 error = blkdev_issue_discard(bdev, start >> 9, len >> 9,
2020 GFP_KERNEL, 0);
2021 break;
2022 default:
2023 return -EOPNOTSUPP;
2024 }
2025 if (error)
2026 return error;
2027
2028
2029
2030
2031
2032
2033 return invalidate_inode_pages2_range(mapping,
2034 start >> PAGE_SHIFT,
2035 end >> PAGE_SHIFT);
2036}
2037
2038const struct file_operations def_blk_fops = {
2039 .open = blkdev_open,
2040 .release = blkdev_close,
2041 .llseek = block_llseek,
2042 .read_iter = blkdev_read_iter,
2043 .write_iter = blkdev_write_iter,
2044 .mmap = generic_file_mmap,
2045 .fsync = blkdev_fsync,
2046 .unlocked_ioctl = block_ioctl,
2047#ifdef CONFIG_COMPAT
2048 .compat_ioctl = compat_blkdev_ioctl,
2049#endif
2050 .splice_read = generic_file_splice_read,
2051 .splice_write = iter_file_splice_write,
2052 .fallocate = blkdev_fallocate,
2053};
2054
2055int ioctl_by_bdev(struct block_device *bdev, unsigned cmd, unsigned long arg)
2056{
2057 int res;
2058 mm_segment_t old_fs = get_fs();
2059 set_fs(KERNEL_DS);
2060 res = blkdev_ioctl(bdev, 0, cmd, arg);
2061 set_fs(old_fs);
2062 return res;
2063}
2064
2065EXPORT_SYMBOL(ioctl_by_bdev);
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075struct block_device *lookup_bdev(const char *pathname)
2076{
2077 struct block_device *bdev;
2078 struct inode *inode;
2079 struct path path;
2080 int error;
2081
2082 if (!pathname || !*pathname)
2083 return ERR_PTR(-EINVAL);
2084
2085 error = kern_path(pathname, LOOKUP_FOLLOW, &path);
2086 if (error)
2087 return ERR_PTR(error);
2088
2089 inode = d_backing_inode(path.dentry);
2090 error = -ENOTBLK;
2091 if (!S_ISBLK(inode->i_mode))
2092 goto fail;
2093 error = -EACCES;
2094 if (!may_open_dev(&path))
2095 goto fail;
2096 error = -ENOMEM;
2097 bdev = bd_acquire(inode);
2098 if (!bdev)
2099 goto fail;
2100out:
2101 path_put(&path);
2102 return bdev;
2103fail:
2104 bdev = ERR_PTR(error);
2105 goto out;
2106}
2107EXPORT_SYMBOL(lookup_bdev);
2108
2109int __invalidate_device(struct block_device *bdev, bool kill_dirty)
2110{
2111 struct super_block *sb = get_super(bdev);
2112 int res = 0;
2113
2114 if (sb) {
2115
2116
2117
2118
2119
2120
2121 shrink_dcache_sb(sb);
2122 res = invalidate_inodes(sb, kill_dirty);
2123 drop_super(sb);
2124 }
2125 invalidate_bdev(bdev);
2126 return res;
2127}
2128EXPORT_SYMBOL(__invalidate_device);
2129
2130void iterate_bdevs(void (*func)(struct block_device *, void *), void *arg)
2131{
2132 struct inode *inode, *old_inode = NULL;
2133
2134 spin_lock(&blockdev_superblock->s_inode_list_lock);
2135 list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list) {
2136 struct address_space *mapping = inode->i_mapping;
2137 struct block_device *bdev;
2138
2139 spin_lock(&inode->i_lock);
2140 if (inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW) ||
2141 mapping->nrpages == 0) {
2142 spin_unlock(&inode->i_lock);
2143 continue;
2144 }
2145 __iget(inode);
2146 spin_unlock(&inode->i_lock);
2147 spin_unlock(&blockdev_superblock->s_inode_list_lock);
2148
2149
2150
2151
2152
2153
2154
2155
2156 iput(old_inode);
2157 old_inode = inode;
2158 bdev = I_BDEV(inode);
2159
2160 mutex_lock(&bdev->bd_mutex);
2161 if (bdev->bd_openers)
2162 func(bdev, arg);
2163 mutex_unlock(&bdev->bd_mutex);
2164
2165 spin_lock(&blockdev_superblock->s_inode_list_lock);
2166 }
2167 spin_unlock(&blockdev_superblock->s_inode_list_lock);
2168 iput(old_inode);
2169}
2170