1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18#include <linux/blkdev.h>
19#include <linux/module.h>
20#include <linux/errno.h>
21#include <linux/slab.h>
22#include <linux/init.h>
23#include <linux/timer.h>
24#include <linux/sched.h>
25#include <linux/list.h>
26#include <linux/file.h>
27#include <linux/mount.h>
28#include <linux/buffer_head.h>
29#include "md.h"
30#include "bitmap.h"
31
32#include <linux/dm-dirty-log.h>
33
34
35#define DEBUG 0
36
37#if DEBUG
38
39
40
41#define INJECT_FAULTS_1 0
42#define INJECT_FAULTS_2 0
43#define INJECT_FAULTS_3 0
44#define INJECT_FAULTS_4 0
45#define INJECT_FAULTS_5 0
46#define INJECT_FAULTS_6 0
47
48
49#define INJECT_FATAL_FAULT_1 0
50#define INJECT_FATAL_FAULT_2 0
51#define INJECT_FATAL_FAULT_3 0
52#endif
53
54#ifndef PRINTK
55# if DEBUG > 0
56# define PRINTK(x...) printk(KERN_DEBUG x)
57# else
58# define PRINTK(x...)
59# endif
60#endif
61
62static inline char *bmname(struct bitmap *bitmap)
63{
64 return bitmap->mddev ? mdname(bitmap->mddev) : "mdX";
65}
66
67
68
69
70static unsigned char *bitmap_alloc_page(struct bitmap *bitmap)
71{
72 unsigned char *page;
73
74#ifdef INJECT_FAULTS_1
75 page = NULL;
76#else
77 page = kzalloc(PAGE_SIZE, GFP_NOIO);
78#endif
79 if (!page)
80 printk("%s: bitmap_alloc_page FAILED\n", bmname(bitmap));
81 else
82 PRINTK("%s: bitmap_alloc_page: allocated page at %p\n",
83 bmname(bitmap), page);
84 return page;
85}
86
87
88
89
90static void bitmap_free_page(struct bitmap *bitmap, unsigned char *page)
91{
92 PRINTK("%s: bitmap_free_page: free page %p\n", bmname(bitmap), page);
93 kfree(page);
94}
95
96
97
98
99
100
101
102
103
104
105
106static int bitmap_checkpage(struct bitmap *bitmap,
107 unsigned long page, int create)
108__releases(bitmap->lock)
109__acquires(bitmap->lock)
110{
111 unsigned char *mappage;
112
113 if (page >= bitmap->pages) {
114
115
116
117
118 return -EINVAL;
119 }
120
121 if (bitmap->bp[page].hijacked)
122 return 0;
123
124 if (bitmap->bp[page].map)
125 return 0;
126
127 if (!create)
128 return -ENOENT;
129
130
131
132 spin_unlock_irq(&bitmap->lock);
133 mappage = bitmap_alloc_page(bitmap);
134 spin_lock_irq(&bitmap->lock);
135
136 if (mappage == NULL) {
137 PRINTK("%s: bitmap map page allocation failed, hijacking\n",
138 bmname(bitmap));
139
140
141 if (!bitmap->bp[page].map)
142 bitmap->bp[page].hijacked = 1;
143 } else if (bitmap->bp[page].map ||
144 bitmap->bp[page].hijacked) {
145
146 bitmap_free_page(bitmap, mappage);
147 return 0;
148 } else {
149
150
151
152 bitmap->bp[page].map = mappage;
153 bitmap->missing_pages--;
154 }
155 return 0;
156}
157
158
159
160
161static void bitmap_checkfree(struct bitmap *bitmap, unsigned long page)
162{
163 char *ptr;
164
165 if (bitmap->bp[page].count)
166 return;
167
168
169
170 if (bitmap->bp[page].hijacked) {
171 bitmap->bp[page].hijacked = 0;
172 bitmap->bp[page].map = NULL;
173 } else {
174
175 ptr = bitmap->bp[page].map;
176 bitmap->bp[page].map = NULL;
177 bitmap->missing_pages++;
178 bitmap_free_page(bitmap, ptr);
179 }
180}
181
182
183
184
185
186
187
188
189
190
191static struct page *read_sb_page(mddev_t *mddev, loff_t offset,
192 struct page *page,
193 unsigned long index, int size)
194{
195
196
197 mdk_rdev_t *rdev;
198 sector_t target;
199 int did_alloc = 0;
200
201 if (!page) {
202 page = alloc_page(GFP_KERNEL);
203 if (!page)
204 return ERR_PTR(-ENOMEM);
205 did_alloc = 1;
206 }
207
208 list_for_each_entry(rdev, &mddev->disks, same_set) {
209 if (! test_bit(In_sync, &rdev->flags)
210 || test_bit(Faulty, &rdev->flags))
211 continue;
212
213 target = offset + index * (PAGE_SIZE/512);
214
215 if (sync_page_io(rdev, target,
216 roundup(size, bdev_logical_block_size(rdev->bdev)),
217 page, READ, true)) {
218 page->index = index;
219 attach_page_buffers(page, NULL);
220
221 return page;
222 }
223 }
224 if (did_alloc)
225 put_page(page);
226 return ERR_PTR(-EIO);
227
228}
229
230static mdk_rdev_t *next_active_rdev(mdk_rdev_t *rdev, mddev_t *mddev)
231{
232
233
234
235
236
237
238
239
240 struct list_head *pos;
241 rcu_read_lock();
242 if (rdev == NULL)
243
244 pos = &mddev->disks;
245 else {
246
247 rdev_dec_pending(rdev, mddev);
248 pos = &rdev->same_set;
249 }
250 list_for_each_continue_rcu(pos, &mddev->disks) {
251 rdev = list_entry(pos, mdk_rdev_t, same_set);
252 if (rdev->raid_disk >= 0 &&
253 !test_bit(Faulty, &rdev->flags)) {
254
255 atomic_inc(&rdev->nr_pending);
256 rcu_read_unlock();
257 return rdev;
258 }
259 }
260 rcu_read_unlock();
261 return NULL;
262}
263
264static int write_sb_page(struct bitmap *bitmap, struct page *page, int wait)
265{
266 mdk_rdev_t *rdev = NULL;
267 struct block_device *bdev;
268 mddev_t *mddev = bitmap->mddev;
269
270 while ((rdev = next_active_rdev(rdev, mddev)) != NULL) {
271 int size = PAGE_SIZE;
272 loff_t offset = mddev->bitmap_info.offset;
273
274 bdev = (rdev->meta_bdev) ? rdev->meta_bdev : rdev->bdev;
275
276 if (page->index == bitmap->file_pages-1)
277 size = roundup(bitmap->last_page_size,
278 bdev_logical_block_size(bdev));
279
280
281
282 if (mddev->external) {
283
284 if (rdev->sb_start + offset + (page->index
285 * (PAGE_SIZE/512))
286 > rdev->data_offset
287 &&
288 rdev->sb_start + offset
289 < (rdev->data_offset + mddev->dev_sectors
290 + (PAGE_SIZE/512)))
291 goto bad_alignment;
292 } else if (offset < 0) {
293
294 if (offset
295 + (long)(page->index * (PAGE_SIZE/512))
296 + size/512 > 0)
297
298 goto bad_alignment;
299 if (rdev->data_offset + mddev->dev_sectors
300 > rdev->sb_start + offset)
301
302 goto bad_alignment;
303 } else if (rdev->sb_start < rdev->data_offset) {
304
305 if (rdev->sb_start
306 + offset
307 + page->index*(PAGE_SIZE/512) + size/512
308 > rdev->data_offset)
309
310 goto bad_alignment;
311 } else {
312
313 }
314 md_super_write(mddev, rdev,
315 rdev->sb_start + offset
316 + page->index * (PAGE_SIZE/512),
317 size,
318 page);
319 }
320
321 if (wait)
322 md_super_wait(mddev);
323 return 0;
324
325 bad_alignment:
326 return -EINVAL;
327}
328
329static void bitmap_file_kick(struct bitmap *bitmap);
330
331
332
333static void write_page(struct bitmap *bitmap, struct page *page, int wait)
334{
335 struct buffer_head *bh;
336
337 if (bitmap->file == NULL) {
338 switch (write_sb_page(bitmap, page, wait)) {
339 case -EINVAL:
340 bitmap->flags |= BITMAP_WRITE_ERROR;
341 }
342 } else {
343
344 bh = page_buffers(page);
345
346 while (bh && bh->b_blocknr) {
347 atomic_inc(&bitmap->pending_writes);
348 set_buffer_locked(bh);
349 set_buffer_mapped(bh);
350 submit_bh(WRITE | REQ_SYNC, bh);
351 bh = bh->b_this_page;
352 }
353
354 if (wait)
355 wait_event(bitmap->write_wait,
356 atomic_read(&bitmap->pending_writes)==0);
357 }
358 if (bitmap->flags & BITMAP_WRITE_ERROR)
359 bitmap_file_kick(bitmap);
360}
361
362static void end_bitmap_write(struct buffer_head *bh, int uptodate)
363{
364 struct bitmap *bitmap = bh->b_private;
365 unsigned long flags;
366
367 if (!uptodate) {
368 spin_lock_irqsave(&bitmap->lock, flags);
369 bitmap->flags |= BITMAP_WRITE_ERROR;
370 spin_unlock_irqrestore(&bitmap->lock, flags);
371 }
372 if (atomic_dec_and_test(&bitmap->pending_writes))
373 wake_up(&bitmap->write_wait);
374}
375
376
377static void
378__clear_page_buffers(struct page *page)
379{
380 ClearPagePrivate(page);
381 set_page_private(page, 0);
382 page_cache_release(page);
383}
384static void free_buffers(struct page *page)
385{
386 struct buffer_head *bh = page_buffers(page);
387
388 while (bh) {
389 struct buffer_head *next = bh->b_this_page;
390 free_buffer_head(bh);
391 bh = next;
392 }
393 __clear_page_buffers(page);
394 put_page(page);
395}
396
397
398
399
400
401
402
403
404static struct page *read_page(struct file *file, unsigned long index,
405 struct bitmap *bitmap,
406 unsigned long count)
407{
408 struct page *page = NULL;
409 struct inode *inode = file->f_path.dentry->d_inode;
410 struct buffer_head *bh;
411 sector_t block;
412
413 PRINTK("read bitmap file (%dB @ %llu)\n", (int)PAGE_SIZE,
414 (unsigned long long)index << PAGE_SHIFT);
415
416 page = alloc_page(GFP_KERNEL);
417 if (!page)
418 page = ERR_PTR(-ENOMEM);
419 if (IS_ERR(page))
420 goto out;
421
422 bh = alloc_page_buffers(page, 1<<inode->i_blkbits, 0);
423 if (!bh) {
424 put_page(page);
425 page = ERR_PTR(-ENOMEM);
426 goto out;
427 }
428 attach_page_buffers(page, bh);
429 block = index << (PAGE_SHIFT - inode->i_blkbits);
430 while (bh) {
431 if (count == 0)
432 bh->b_blocknr = 0;
433 else {
434 bh->b_blocknr = bmap(inode, block);
435 if (bh->b_blocknr == 0) {
436
437 free_buffers(page);
438 page = ERR_PTR(-EINVAL);
439 goto out;
440 }
441 bh->b_bdev = inode->i_sb->s_bdev;
442 if (count < (1<<inode->i_blkbits))
443 count = 0;
444 else
445 count -= (1<<inode->i_blkbits);
446
447 bh->b_end_io = end_bitmap_write;
448 bh->b_private = bitmap;
449 atomic_inc(&bitmap->pending_writes);
450 set_buffer_locked(bh);
451 set_buffer_mapped(bh);
452 submit_bh(READ, bh);
453 }
454 block++;
455 bh = bh->b_this_page;
456 }
457 page->index = index;
458
459 wait_event(bitmap->write_wait,
460 atomic_read(&bitmap->pending_writes)==0);
461 if (bitmap->flags & BITMAP_WRITE_ERROR) {
462 free_buffers(page);
463 page = ERR_PTR(-EIO);
464 }
465out:
466 if (IS_ERR(page))
467 printk(KERN_ALERT "md: bitmap read error: (%dB @ %llu): %ld\n",
468 (int)PAGE_SIZE,
469 (unsigned long long)index << PAGE_SHIFT,
470 PTR_ERR(page));
471 return page;
472}
473
474
475
476
477
478
479void bitmap_update_sb(struct bitmap *bitmap)
480{
481 bitmap_super_t *sb;
482 unsigned long flags;
483
484 if (!bitmap || !bitmap->mddev)
485 return;
486 if (bitmap->mddev->bitmap_info.external)
487 return;
488 spin_lock_irqsave(&bitmap->lock, flags);
489 if (!bitmap->sb_page) {
490 spin_unlock_irqrestore(&bitmap->lock, flags);
491 return;
492 }
493 spin_unlock_irqrestore(&bitmap->lock, flags);
494 sb = kmap_atomic(bitmap->sb_page, KM_USER0);
495 sb->events = cpu_to_le64(bitmap->mddev->events);
496 if (bitmap->mddev->events < bitmap->events_cleared)
497
498 bitmap->events_cleared = bitmap->mddev->events;
499 sb->events_cleared = cpu_to_le64(bitmap->events_cleared);
500 sb->state = cpu_to_le32(bitmap->flags);
501
502 sb->daemon_sleep = cpu_to_le32(bitmap->mddev->bitmap_info.daemon_sleep/HZ);
503 sb->write_behind = cpu_to_le32(bitmap->mddev->bitmap_info.max_write_behind);
504 kunmap_atomic(sb, KM_USER0);
505 write_page(bitmap, bitmap->sb_page, 1);
506}
507
508
509void bitmap_print_sb(struct bitmap *bitmap)
510{
511 bitmap_super_t *sb;
512
513 if (!bitmap || !bitmap->sb_page)
514 return;
515 sb = kmap_atomic(bitmap->sb_page, KM_USER0);
516 printk(KERN_DEBUG "%s: bitmap file superblock:\n", bmname(bitmap));
517 printk(KERN_DEBUG " magic: %08x\n", le32_to_cpu(sb->magic));
518 printk(KERN_DEBUG " version: %d\n", le32_to_cpu(sb->version));
519 printk(KERN_DEBUG " uuid: %08x.%08x.%08x.%08x\n",
520 *(__u32 *)(sb->uuid+0),
521 *(__u32 *)(sb->uuid+4),
522 *(__u32 *)(sb->uuid+8),
523 *(__u32 *)(sb->uuid+12));
524 printk(KERN_DEBUG " events: %llu\n",
525 (unsigned long long) le64_to_cpu(sb->events));
526 printk(KERN_DEBUG "events cleared: %llu\n",
527 (unsigned long long) le64_to_cpu(sb->events_cleared));
528 printk(KERN_DEBUG " state: %08x\n", le32_to_cpu(sb->state));
529 printk(KERN_DEBUG " chunksize: %d B\n", le32_to_cpu(sb->chunksize));
530 printk(KERN_DEBUG " daemon sleep: %ds\n", le32_to_cpu(sb->daemon_sleep));
531 printk(KERN_DEBUG " sync size: %llu KB\n",
532 (unsigned long long)le64_to_cpu(sb->sync_size)/2);
533 printk(KERN_DEBUG "max write behind: %d\n", le32_to_cpu(sb->write_behind));
534 kunmap_atomic(sb, KM_USER0);
535}
536
537
538
539
540
541
542
543
544
545
546
547
548static int bitmap_new_disk_sb(struct bitmap *bitmap)
549{
550 bitmap_super_t *sb;
551 unsigned long chunksize, daemon_sleep, write_behind;
552 int err = -EINVAL;
553
554 bitmap->sb_page = alloc_page(GFP_KERNEL);
555 if (IS_ERR(bitmap->sb_page)) {
556 err = PTR_ERR(bitmap->sb_page);
557 bitmap->sb_page = NULL;
558 return err;
559 }
560 bitmap->sb_page->index = 0;
561
562 sb = kmap_atomic(bitmap->sb_page, KM_USER0);
563
564 sb->magic = cpu_to_le32(BITMAP_MAGIC);
565 sb->version = cpu_to_le32(BITMAP_MAJOR_HI);
566
567 chunksize = bitmap->mddev->bitmap_info.chunksize;
568 BUG_ON(!chunksize);
569 if (!is_power_of_2(chunksize)) {
570 kunmap_atomic(sb, KM_USER0);
571 printk(KERN_ERR "bitmap chunksize not a power of 2\n");
572 return -EINVAL;
573 }
574 sb->chunksize = cpu_to_le32(chunksize);
575
576 daemon_sleep = bitmap->mddev->bitmap_info.daemon_sleep;
577 if (!daemon_sleep ||
578 (daemon_sleep < 1) || (daemon_sleep > MAX_SCHEDULE_TIMEOUT)) {
579 printk(KERN_INFO "Choosing daemon_sleep default (5 sec)\n");
580 daemon_sleep = 5 * HZ;
581 }
582 sb->daemon_sleep = cpu_to_le32(daemon_sleep);
583 bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;
584
585
586
587
588
589 write_behind = bitmap->mddev->bitmap_info.max_write_behind;
590 if (write_behind > COUNTER_MAX)
591 write_behind = COUNTER_MAX / 2;
592 sb->write_behind = cpu_to_le32(write_behind);
593 bitmap->mddev->bitmap_info.max_write_behind = write_behind;
594
595
596 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
597
598 memcpy(sb->uuid, bitmap->mddev->uuid, 16);
599
600 bitmap->flags |= BITMAP_STALE;
601 sb->state |= cpu_to_le32(BITMAP_STALE);
602 bitmap->events_cleared = bitmap->mddev->events;
603 sb->events_cleared = cpu_to_le64(bitmap->mddev->events);
604
605 bitmap->flags |= BITMAP_HOSTENDIAN;
606 sb->version = cpu_to_le32(BITMAP_MAJOR_HOSTENDIAN);
607
608 kunmap_atomic(sb, KM_USER0);
609
610 return 0;
611}
612
613
614static int bitmap_read_sb(struct bitmap *bitmap)
615{
616 char *reason = NULL;
617 bitmap_super_t *sb;
618 unsigned long chunksize, daemon_sleep, write_behind;
619 unsigned long long events;
620 int err = -EINVAL;
621
622
623 if (bitmap->file) {
624 loff_t isize = i_size_read(bitmap->file->f_mapping->host);
625 int bytes = isize > PAGE_SIZE ? PAGE_SIZE : isize;
626
627 bitmap->sb_page = read_page(bitmap->file, 0, bitmap, bytes);
628 } else {
629 bitmap->sb_page = read_sb_page(bitmap->mddev,
630 bitmap->mddev->bitmap_info.offset,
631 NULL,
632 0, sizeof(bitmap_super_t));
633 }
634 if (IS_ERR(bitmap->sb_page)) {
635 err = PTR_ERR(bitmap->sb_page);
636 bitmap->sb_page = NULL;
637 return err;
638 }
639
640 sb = kmap_atomic(bitmap->sb_page, KM_USER0);
641
642 chunksize = le32_to_cpu(sb->chunksize);
643 daemon_sleep = le32_to_cpu(sb->daemon_sleep) * HZ;
644 write_behind = le32_to_cpu(sb->write_behind);
645
646
647 if (sb->magic != cpu_to_le32(BITMAP_MAGIC))
648 reason = "bad magic";
649 else if (le32_to_cpu(sb->version) < BITMAP_MAJOR_LO ||
650 le32_to_cpu(sb->version) > BITMAP_MAJOR_HI)
651 reason = "unrecognized superblock version";
652 else if (chunksize < 512)
653 reason = "bitmap chunksize too small";
654 else if (!is_power_of_2(chunksize))
655 reason = "bitmap chunksize not a power of 2";
656 else if (daemon_sleep < 1 || daemon_sleep > MAX_SCHEDULE_TIMEOUT)
657 reason = "daemon sleep period out of range";
658 else if (write_behind > COUNTER_MAX)
659 reason = "write-behind limit out of range (0 - 16383)";
660 if (reason) {
661 printk(KERN_INFO "%s: invalid bitmap file superblock: %s\n",
662 bmname(bitmap), reason);
663 goto out;
664 }
665
666
667 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
668
669 if (!bitmap->mddev->persistent)
670 goto success;
671
672
673
674
675
676 if (memcmp(sb->uuid, bitmap->mddev->uuid, 16)) {
677 printk(KERN_INFO "%s: bitmap superblock UUID mismatch\n",
678 bmname(bitmap));
679 goto out;
680 }
681 events = le64_to_cpu(sb->events);
682 if (events < bitmap->mddev->events) {
683 printk(KERN_INFO "%s: bitmap file is out of date (%llu < %llu) "
684 "-- forcing full recovery\n", bmname(bitmap), events,
685 (unsigned long long) bitmap->mddev->events);
686 sb->state |= cpu_to_le32(BITMAP_STALE);
687 }
688success:
689
690 bitmap->mddev->bitmap_info.chunksize = chunksize;
691 bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;
692 bitmap->mddev->bitmap_info.max_write_behind = write_behind;
693 bitmap->flags |= le32_to_cpu(sb->state);
694 if (le32_to_cpu(sb->version) == BITMAP_MAJOR_HOSTENDIAN)
695 bitmap->flags |= BITMAP_HOSTENDIAN;
696 bitmap->events_cleared = le64_to_cpu(sb->events_cleared);
697 if (bitmap->flags & BITMAP_STALE)
698 bitmap->events_cleared = bitmap->mddev->events;
699 err = 0;
700out:
701 kunmap_atomic(sb, KM_USER0);
702 if (err)
703 bitmap_print_sb(bitmap);
704 return err;
705}
706
707enum bitmap_mask_op {
708 MASK_SET,
709 MASK_UNSET
710};
711
712
713static int bitmap_mask_state(struct bitmap *bitmap, enum bitmap_state bits,
714 enum bitmap_mask_op op)
715{
716 bitmap_super_t *sb;
717 unsigned long flags;
718 int old;
719
720 spin_lock_irqsave(&bitmap->lock, flags);
721 if (!bitmap->sb_page) {
722 spin_unlock_irqrestore(&bitmap->lock, flags);
723 return 0;
724 }
725 spin_unlock_irqrestore(&bitmap->lock, flags);
726 sb = kmap_atomic(bitmap->sb_page, KM_USER0);
727 old = le32_to_cpu(sb->state) & bits;
728 switch (op) {
729 case MASK_SET:
730 sb->state |= cpu_to_le32(bits);
731 bitmap->flags |= bits;
732 break;
733 case MASK_UNSET:
734 sb->state &= cpu_to_le32(~bits);
735 bitmap->flags &= ~bits;
736 break;
737 default:
738 BUG();
739 }
740 kunmap_atomic(sb, KM_USER0);
741 return old;
742}
743
744
745
746
747
748
749
750
751
752
753
754
755static inline unsigned long file_page_index(struct bitmap *bitmap, unsigned long chunk)
756{
757 if (!bitmap->mddev->bitmap_info.external)
758 chunk += sizeof(bitmap_super_t) << 3;
759 return chunk >> PAGE_BIT_SHIFT;
760}
761
762
763static inline unsigned long file_page_offset(struct bitmap *bitmap, unsigned long chunk)
764{
765 if (!bitmap->mddev->bitmap_info.external)
766 chunk += sizeof(bitmap_super_t) << 3;
767 return chunk & (PAGE_BITS - 1);
768}
769
770
771
772
773
774
775
776
777static inline struct page *filemap_get_page(struct bitmap *bitmap,
778 unsigned long chunk)
779{
780 if (bitmap->filemap == NULL)
781 return NULL;
782 if (file_page_index(bitmap, chunk) >= bitmap->file_pages)
783 return NULL;
784 return bitmap->filemap[file_page_index(bitmap, chunk)
785 - file_page_index(bitmap, 0)];
786}
787
788static void bitmap_file_unmap(struct bitmap *bitmap)
789{
790 struct page **map, *sb_page;
791 unsigned long *attr;
792 int pages;
793 unsigned long flags;
794
795 spin_lock_irqsave(&bitmap->lock, flags);
796 map = bitmap->filemap;
797 bitmap->filemap = NULL;
798 attr = bitmap->filemap_attr;
799 bitmap->filemap_attr = NULL;
800 pages = bitmap->file_pages;
801 bitmap->file_pages = 0;
802 sb_page = bitmap->sb_page;
803 bitmap->sb_page = NULL;
804 spin_unlock_irqrestore(&bitmap->lock, flags);
805
806 while (pages--)
807 if (map[pages] != sb_page)
808 free_buffers(map[pages]);
809 kfree(map);
810 kfree(attr);
811
812 if (sb_page)
813 free_buffers(sb_page);
814}
815
816static void bitmap_file_put(struct bitmap *bitmap)
817{
818 struct file *file;
819 unsigned long flags;
820
821 spin_lock_irqsave(&bitmap->lock, flags);
822 file = bitmap->file;
823 bitmap->file = NULL;
824 spin_unlock_irqrestore(&bitmap->lock, flags);
825
826 if (file)
827 wait_event(bitmap->write_wait,
828 atomic_read(&bitmap->pending_writes)==0);
829 bitmap_file_unmap(bitmap);
830
831 if (file) {
832 struct inode *inode = file->f_path.dentry->d_inode;
833 invalidate_mapping_pages(inode->i_mapping, 0, -1);
834 fput(file);
835 }
836}
837
838
839
840
841
842
843static void bitmap_file_kick(struct bitmap *bitmap)
844{
845 char *path, *ptr = NULL;
846
847 if (bitmap_mask_state(bitmap, BITMAP_STALE, MASK_SET) == 0) {
848 bitmap_update_sb(bitmap);
849
850 if (bitmap->file) {
851 path = kmalloc(PAGE_SIZE, GFP_KERNEL);
852 if (path)
853 ptr = d_path(&bitmap->file->f_path, path,
854 PAGE_SIZE);
855
856 printk(KERN_ALERT
857 "%s: kicking failed bitmap file %s from array!\n",
858 bmname(bitmap), IS_ERR(ptr) ? "" : ptr);
859
860 kfree(path);
861 } else
862 printk(KERN_ALERT
863 "%s: disabling internal bitmap due to errors\n",
864 bmname(bitmap));
865 }
866
867 bitmap_file_put(bitmap);
868
869 return;
870}
871
872enum bitmap_page_attr {
873 BITMAP_PAGE_DIRTY = 0,
874 BITMAP_PAGE_CLEAN = 1,
875 BITMAP_PAGE_NEEDWRITE = 2,
876};
877
878static inline void set_page_attr(struct bitmap *bitmap, struct page *page,
879 enum bitmap_page_attr attr)
880{
881 if (page)
882 __set_bit((page->index<<2) + attr, bitmap->filemap_attr);
883 else
884 __set_bit(attr, &bitmap->logattrs);
885}
886
887static inline void clear_page_attr(struct bitmap *bitmap, struct page *page,
888 enum bitmap_page_attr attr)
889{
890 if (page)
891 __clear_bit((page->index<<2) + attr, bitmap->filemap_attr);
892 else
893 __clear_bit(attr, &bitmap->logattrs);
894}
895
896static inline unsigned long test_page_attr(struct bitmap *bitmap, struct page *page,
897 enum bitmap_page_attr attr)
898{
899 if (page)
900 return test_bit((page->index<<2) + attr, bitmap->filemap_attr);
901 else
902 return test_bit(attr, &bitmap->logattrs);
903}
904
905
906
907
908
909
910
911
912static void bitmap_file_set_bit(struct bitmap *bitmap, sector_t block)
913{
914 unsigned long bit;
915 struct page *page = NULL;
916 void *kaddr;
917 unsigned long chunk = block >> CHUNK_BLOCK_SHIFT(bitmap);
918
919 if (!bitmap->filemap) {
920 struct dm_dirty_log *log = bitmap->mddev->bitmap_info.log;
921 if (log)
922 log->type->mark_region(log, chunk);
923 } else {
924
925 page = filemap_get_page(bitmap, chunk);
926 if (!page)
927 return;
928 bit = file_page_offset(bitmap, chunk);
929
930
931 kaddr = kmap_atomic(page, KM_USER0);
932 if (bitmap->flags & BITMAP_HOSTENDIAN)
933 set_bit(bit, kaddr);
934 else
935 __test_and_set_bit_le(bit, kaddr);
936 kunmap_atomic(kaddr, KM_USER0);
937 PRINTK("set file bit %lu page %lu\n", bit, page->index);
938 }
939
940 set_page_attr(bitmap, page, BITMAP_PAGE_DIRTY);
941}
942
943
944
945
946void bitmap_unplug(struct bitmap *bitmap)
947{
948 unsigned long i, flags;
949 int dirty, need_write;
950 struct page *page;
951 int wait = 0;
952
953 if (!bitmap)
954 return;
955 if (!bitmap->filemap) {
956
957 struct dm_dirty_log *log = bitmap->mddev->bitmap_info.log;
958 dirty = test_and_clear_bit(BITMAP_PAGE_DIRTY, &bitmap->logattrs);
959 need_write = test_and_clear_bit(BITMAP_PAGE_NEEDWRITE, &bitmap->logattrs);
960 if (dirty || need_write)
961 if (log->type->flush(log))
962 bitmap->flags |= BITMAP_WRITE_ERROR;
963 goto out;
964 }
965
966
967
968 for (i = 0; i < bitmap->file_pages; i++) {
969 spin_lock_irqsave(&bitmap->lock, flags);
970 if (!bitmap->filemap) {
971 spin_unlock_irqrestore(&bitmap->lock, flags);
972 return;
973 }
974 page = bitmap->filemap[i];
975 dirty = test_page_attr(bitmap, page, BITMAP_PAGE_DIRTY);
976 need_write = test_page_attr(bitmap, page, BITMAP_PAGE_NEEDWRITE);
977 clear_page_attr(bitmap, page, BITMAP_PAGE_DIRTY);
978 clear_page_attr(bitmap, page, BITMAP_PAGE_NEEDWRITE);
979 if (dirty)
980 wait = 1;
981 spin_unlock_irqrestore(&bitmap->lock, flags);
982
983 if (dirty || need_write)
984 write_page(bitmap, page, 0);
985 }
986 if (wait) {
987 if (bitmap->file)
988 wait_event(bitmap->write_wait,
989 atomic_read(&bitmap->pending_writes)==0);
990 else
991 md_super_wait(bitmap->mddev);
992 }
993out:
994 if (bitmap->flags & BITMAP_WRITE_ERROR)
995 bitmap_file_kick(bitmap);
996}
997EXPORT_SYMBOL(bitmap_unplug);
998
999static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed);
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011static int bitmap_init_from_disk(struct bitmap *bitmap, sector_t start)
1012{
1013 unsigned long i, chunks, index, oldindex, bit;
1014 struct page *page = NULL, *oldpage = NULL;
1015 unsigned long num_pages, bit_cnt = 0;
1016 struct file *file;
1017 unsigned long bytes, offset;
1018 int outofdate;
1019 int ret = -ENOSPC;
1020 void *paddr;
1021
1022 chunks = bitmap->chunks;
1023 file = bitmap->file;
1024
1025 BUG_ON(!file && !bitmap->mddev->bitmap_info.offset);
1026
1027#ifdef INJECT_FAULTS_3
1028 outofdate = 1;
1029#else
1030 outofdate = bitmap->flags & BITMAP_STALE;
1031#endif
1032 if (outofdate)
1033 printk(KERN_INFO "%s: bitmap file is out of date, doing full "
1034 "recovery\n", bmname(bitmap));
1035
1036 bytes = DIV_ROUND_UP(bitmap->chunks, 8);
1037 if (!bitmap->mddev->bitmap_info.external)
1038 bytes += sizeof(bitmap_super_t);
1039
1040 num_pages = DIV_ROUND_UP(bytes, PAGE_SIZE);
1041
1042 if (file && i_size_read(file->f_mapping->host) < bytes) {
1043 printk(KERN_INFO "%s: bitmap file too short %lu < %lu\n",
1044 bmname(bitmap),
1045 (unsigned long) i_size_read(file->f_mapping->host),
1046 bytes);
1047 goto err;
1048 }
1049
1050 ret = -ENOMEM;
1051
1052 bitmap->filemap = kmalloc(sizeof(struct page *) * num_pages, GFP_KERNEL);
1053 if (!bitmap->filemap)
1054 goto err;
1055
1056
1057 bitmap->filemap_attr = kzalloc(
1058 roundup(DIV_ROUND_UP(num_pages*4, 8), sizeof(unsigned long)),
1059 GFP_KERNEL);
1060 if (!bitmap->filemap_attr)
1061 goto err;
1062
1063 oldindex = ~0L;
1064
1065 for (i = 0; i < chunks; i++) {
1066 int b;
1067 index = file_page_index(bitmap, i);
1068 bit = file_page_offset(bitmap, i);
1069 if (index != oldindex) {
1070 int count;
1071
1072 if (index == num_pages-1)
1073 count = bytes - index * PAGE_SIZE;
1074 else
1075 count = PAGE_SIZE;
1076 if (index == 0 && bitmap->sb_page) {
1077
1078
1079
1080
1081
1082 page = bitmap->sb_page;
1083 offset = sizeof(bitmap_super_t);
1084 if (!file)
1085 page = read_sb_page(
1086 bitmap->mddev,
1087 bitmap->mddev->bitmap_info.offset,
1088 page,
1089 index, count);
1090 } else if (file) {
1091 page = read_page(file, index, bitmap, count);
1092 offset = 0;
1093 } else {
1094 page = read_sb_page(bitmap->mddev,
1095 bitmap->mddev->bitmap_info.offset,
1096 NULL,
1097 index, count);
1098 offset = 0;
1099 }
1100 if (IS_ERR(page)) {
1101 ret = PTR_ERR(page);
1102 goto err;
1103 }
1104
1105 oldindex = index;
1106 oldpage = page;
1107
1108 bitmap->filemap[bitmap->file_pages++] = page;
1109 bitmap->last_page_size = count;
1110
1111 if (outofdate) {
1112
1113
1114
1115
1116 paddr = kmap_atomic(page, KM_USER0);
1117 memset(paddr + offset, 0xff,
1118 PAGE_SIZE - offset);
1119 kunmap_atomic(paddr, KM_USER0);
1120 write_page(bitmap, page, 1);
1121
1122 ret = -EIO;
1123 if (bitmap->flags & BITMAP_WRITE_ERROR)
1124 goto err;
1125 }
1126 }
1127 paddr = kmap_atomic(page, KM_USER0);
1128 if (bitmap->flags & BITMAP_HOSTENDIAN)
1129 b = test_bit(bit, paddr);
1130 else
1131 b = test_bit_le(bit, paddr);
1132 kunmap_atomic(paddr, KM_USER0);
1133 if (b) {
1134
1135 int needed = ((sector_t)(i+1) << (CHUNK_BLOCK_SHIFT(bitmap))
1136 >= start);
1137 bitmap_set_memory_bits(bitmap,
1138 (sector_t)i << CHUNK_BLOCK_SHIFT(bitmap),
1139 needed);
1140 bit_cnt++;
1141 set_page_attr(bitmap, page, BITMAP_PAGE_CLEAN);
1142 }
1143 }
1144
1145
1146 ret = 0;
1147 bitmap_mask_state(bitmap, BITMAP_STALE, MASK_UNSET);
1148
1149 if (bit_cnt) {
1150 set_bit(MD_RECOVERY_NEEDED, &bitmap->mddev->recovery);
1151 md_wakeup_thread(bitmap->mddev->thread);
1152 }
1153
1154 printk(KERN_INFO "%s: bitmap initialized from disk: "
1155 "read %lu/%lu pages, set %lu of %lu bits\n",
1156 bmname(bitmap), bitmap->file_pages, num_pages, bit_cnt, chunks);
1157
1158 return 0;
1159
1160 err:
1161 printk(KERN_INFO "%s: bitmap initialisation failed: %d\n",
1162 bmname(bitmap), ret);
1163 return ret;
1164}
1165
1166void bitmap_write_all(struct bitmap *bitmap)
1167{
1168
1169
1170
1171 int i;
1172
1173 for (i = 0; i < bitmap->file_pages; i++)
1174 set_page_attr(bitmap, bitmap->filemap[i],
1175 BITMAP_PAGE_NEEDWRITE);
1176}
1177
1178static void bitmap_count_page(struct bitmap *bitmap, sector_t offset, int inc)
1179{
1180 sector_t chunk = offset >> CHUNK_BLOCK_SHIFT(bitmap);
1181 unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1182 bitmap->bp[page].count += inc;
1183 bitmap_checkfree(bitmap, page);
1184}
1185static bitmap_counter_t *bitmap_get_counter(struct bitmap *bitmap,
1186 sector_t offset, sector_t *blocks,
1187 int create);
1188
1189
1190
1191
1192
1193
1194void bitmap_daemon_work(mddev_t *mddev)
1195{
1196 struct bitmap *bitmap;
1197 unsigned long j;
1198 unsigned long flags;
1199 struct page *page = NULL, *lastpage = NULL;
1200 sector_t blocks;
1201 void *paddr;
1202 struct dm_dirty_log *log = mddev->bitmap_info.log;
1203
1204
1205
1206
1207 mutex_lock(&mddev->bitmap_info.mutex);
1208 bitmap = mddev->bitmap;
1209 if (bitmap == NULL) {
1210 mutex_unlock(&mddev->bitmap_info.mutex);
1211 return;
1212 }
1213 if (time_before(jiffies, bitmap->daemon_lastrun
1214 + bitmap->mddev->bitmap_info.daemon_sleep))
1215 goto done;
1216
1217 bitmap->daemon_lastrun = jiffies;
1218 if (bitmap->allclean) {
1219 bitmap->mddev->thread->timeout = MAX_SCHEDULE_TIMEOUT;
1220 goto done;
1221 }
1222 bitmap->allclean = 1;
1223
1224 spin_lock_irqsave(&bitmap->lock, flags);
1225 for (j = 0; j < bitmap->chunks; j++) {
1226 bitmap_counter_t *bmc;
1227 if (!bitmap->filemap) {
1228 if (!log)
1229
1230 break;
1231 } else
1232 page = filemap_get_page(bitmap, j);
1233
1234 if (page != lastpage) {
1235
1236 if (!test_page_attr(bitmap, page, BITMAP_PAGE_CLEAN)) {
1237 int need_write = test_page_attr(bitmap, page,
1238 BITMAP_PAGE_NEEDWRITE);
1239 if (need_write)
1240 clear_page_attr(bitmap, page, BITMAP_PAGE_NEEDWRITE);
1241
1242 spin_unlock_irqrestore(&bitmap->lock, flags);
1243 if (need_write) {
1244 write_page(bitmap, page, 0);
1245 bitmap->allclean = 0;
1246 }
1247 spin_lock_irqsave(&bitmap->lock, flags);
1248 j |= (PAGE_BITS - 1);
1249 continue;
1250 }
1251
1252
1253 if (lastpage != NULL) {
1254 if (test_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE)) {
1255 clear_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE);
1256 spin_unlock_irqrestore(&bitmap->lock, flags);
1257 write_page(bitmap, lastpage, 0);
1258 } else {
1259 set_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE);
1260 spin_unlock_irqrestore(&bitmap->lock, flags);
1261 }
1262 } else
1263 spin_unlock_irqrestore(&bitmap->lock, flags);
1264 lastpage = page;
1265
1266
1267
1268
1269 if (bitmap->need_sync &&
1270 bitmap->mddev->bitmap_info.external == 0) {
1271 bitmap_super_t *sb;
1272 bitmap->need_sync = 0;
1273 sb = kmap_atomic(bitmap->sb_page, KM_USER0);
1274 sb->events_cleared =
1275 cpu_to_le64(bitmap->events_cleared);
1276 kunmap_atomic(sb, KM_USER0);
1277 write_page(bitmap, bitmap->sb_page, 1);
1278 }
1279 spin_lock_irqsave(&bitmap->lock, flags);
1280 if (!bitmap->need_sync)
1281 clear_page_attr(bitmap, page, BITMAP_PAGE_CLEAN);
1282 }
1283 bmc = bitmap_get_counter(bitmap,
1284 (sector_t)j << CHUNK_BLOCK_SHIFT(bitmap),
1285 &blocks, 0);
1286 if (bmc) {
1287 if (*bmc)
1288 bitmap->allclean = 0;
1289
1290 if (*bmc == 2) {
1291 *bmc = 1;
1292 set_page_attr(bitmap, page, BITMAP_PAGE_CLEAN);
1293 } else if (*bmc == 1 && !bitmap->need_sync) {
1294
1295 *bmc = 0;
1296 bitmap_count_page(bitmap,
1297 (sector_t)j << CHUNK_BLOCK_SHIFT(bitmap),
1298 -1);
1299
1300
1301 if (page) {
1302 paddr = kmap_atomic(page, KM_USER0);
1303 if (bitmap->flags & BITMAP_HOSTENDIAN)
1304 clear_bit(file_page_offset(bitmap, j),
1305 paddr);
1306 else
1307 __test_and_clear_bit_le(file_page_offset(bitmap, j),
1308 paddr);
1309 kunmap_atomic(paddr, KM_USER0);
1310 } else
1311 log->type->clear_region(log, j);
1312 }
1313 } else
1314 j |= PAGE_COUNTER_MASK;
1315 }
1316 spin_unlock_irqrestore(&bitmap->lock, flags);
1317
1318
1319 if (lastpage != NULL || log != NULL) {
1320 spin_lock_irqsave(&bitmap->lock, flags);
1321 if (test_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE)) {
1322 clear_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE);
1323 spin_unlock_irqrestore(&bitmap->lock, flags);
1324 if (lastpage)
1325 write_page(bitmap, lastpage, 0);
1326 else
1327 if (log->type->flush(log))
1328 bitmap->flags |= BITMAP_WRITE_ERROR;
1329 } else {
1330 set_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE);
1331 spin_unlock_irqrestore(&bitmap->lock, flags);
1332 }
1333 }
1334
1335 done:
1336 if (bitmap->allclean == 0)
1337 bitmap->mddev->thread->timeout =
1338 bitmap->mddev->bitmap_info.daemon_sleep;
1339 mutex_unlock(&mddev->bitmap_info.mutex);
1340}
1341
1342static bitmap_counter_t *bitmap_get_counter(struct bitmap *bitmap,
1343 sector_t offset, sector_t *blocks,
1344 int create)
1345__releases(bitmap->lock)
1346__acquires(bitmap->lock)
1347{
1348
1349
1350
1351
1352 sector_t chunk = offset >> CHUNK_BLOCK_SHIFT(bitmap);
1353 unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1354 unsigned long pageoff = (chunk & PAGE_COUNTER_MASK) << COUNTER_BYTE_SHIFT;
1355 sector_t csize;
1356 int err;
1357
1358 err = bitmap_checkpage(bitmap, page, create);
1359
1360 if (bitmap->bp[page].hijacked ||
1361 bitmap->bp[page].map == NULL)
1362 csize = ((sector_t)1) << (CHUNK_BLOCK_SHIFT(bitmap) +
1363 PAGE_COUNTER_SHIFT - 1);
1364 else
1365 csize = ((sector_t)1) << (CHUNK_BLOCK_SHIFT(bitmap));
1366 *blocks = csize - (offset & (csize - 1));
1367
1368 if (err < 0)
1369 return NULL;
1370
1371
1372
1373 if (bitmap->bp[page].hijacked) {
1374
1375
1376 int hi = (pageoff > PAGE_COUNTER_MASK);
1377 return &((bitmap_counter_t *)
1378 &bitmap->bp[page].map)[hi];
1379 } else
1380 return (bitmap_counter_t *)
1381 &(bitmap->bp[page].map[pageoff]);
1382}
1383
1384int bitmap_startwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors, int behind)
1385{
1386 if (!bitmap)
1387 return 0;
1388
1389 if (behind) {
1390 int bw;
1391 atomic_inc(&bitmap->behind_writes);
1392 bw = atomic_read(&bitmap->behind_writes);
1393 if (bw > bitmap->behind_writes_used)
1394 bitmap->behind_writes_used = bw;
1395
1396 PRINTK(KERN_DEBUG "inc write-behind count %d/%d\n",
1397 bw, bitmap->max_write_behind);
1398 }
1399
1400 while (sectors) {
1401 sector_t blocks;
1402 bitmap_counter_t *bmc;
1403
1404 spin_lock_irq(&bitmap->lock);
1405 bmc = bitmap_get_counter(bitmap, offset, &blocks, 1);
1406 if (!bmc) {
1407 spin_unlock_irq(&bitmap->lock);
1408 return 0;
1409 }
1410
1411 if (unlikely(COUNTER(*bmc) == COUNTER_MAX)) {
1412 DEFINE_WAIT(__wait);
1413
1414
1415
1416
1417 prepare_to_wait(&bitmap->overflow_wait, &__wait,
1418 TASK_UNINTERRUPTIBLE);
1419 spin_unlock_irq(&bitmap->lock);
1420 io_schedule();
1421 finish_wait(&bitmap->overflow_wait, &__wait);
1422 continue;
1423 }
1424
1425 switch (*bmc) {
1426 case 0:
1427 bitmap_file_set_bit(bitmap, offset);
1428 bitmap_count_page(bitmap, offset, 1);
1429
1430 case 1:
1431 *bmc = 2;
1432 }
1433
1434 (*bmc)++;
1435
1436 spin_unlock_irq(&bitmap->lock);
1437
1438 offset += blocks;
1439 if (sectors > blocks)
1440 sectors -= blocks;
1441 else
1442 sectors = 0;
1443 }
1444 bitmap->allclean = 0;
1445 return 0;
1446}
1447EXPORT_SYMBOL(bitmap_startwrite);
1448
1449void bitmap_endwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors,
1450 int success, int behind)
1451{
1452 if (!bitmap)
1453 return;
1454 if (behind) {
1455 if (atomic_dec_and_test(&bitmap->behind_writes))
1456 wake_up(&bitmap->behind_wait);
1457 PRINTK(KERN_DEBUG "dec write-behind count %d/%d\n",
1458 atomic_read(&bitmap->behind_writes), bitmap->max_write_behind);
1459 }
1460 if (bitmap->mddev->degraded)
1461
1462 success = 0;
1463
1464 while (sectors) {
1465 sector_t blocks;
1466 unsigned long flags;
1467 bitmap_counter_t *bmc;
1468
1469 spin_lock_irqsave(&bitmap->lock, flags);
1470 bmc = bitmap_get_counter(bitmap, offset, &blocks, 0);
1471 if (!bmc) {
1472 spin_unlock_irqrestore(&bitmap->lock, flags);
1473 return;
1474 }
1475
1476 if (success &&
1477 bitmap->events_cleared < bitmap->mddev->events) {
1478 bitmap->events_cleared = bitmap->mddev->events;
1479 bitmap->need_sync = 1;
1480 sysfs_notify_dirent_safe(bitmap->sysfs_can_clear);
1481 }
1482
1483 if (!success && !NEEDED(*bmc))
1484 *bmc |= NEEDED_MASK;
1485
1486 if (COUNTER(*bmc) == COUNTER_MAX)
1487 wake_up(&bitmap->overflow_wait);
1488
1489 (*bmc)--;
1490 if (*bmc <= 2)
1491 set_page_attr(bitmap,
1492 filemap_get_page(
1493 bitmap,
1494 offset >> CHUNK_BLOCK_SHIFT(bitmap)),
1495 BITMAP_PAGE_CLEAN);
1496
1497 spin_unlock_irqrestore(&bitmap->lock, flags);
1498 offset += blocks;
1499 if (sectors > blocks)
1500 sectors -= blocks;
1501 else
1502 sectors = 0;
1503 }
1504}
1505EXPORT_SYMBOL(bitmap_endwrite);
1506
1507static int __bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks,
1508 int degraded)
1509{
1510 bitmap_counter_t *bmc;
1511 int rv;
1512 if (bitmap == NULL) {
1513 *blocks = 1024;
1514 return 1;
1515 }
1516 spin_lock_irq(&bitmap->lock);
1517 bmc = bitmap_get_counter(bitmap, offset, blocks, 0);
1518 rv = 0;
1519 if (bmc) {
1520
1521 if (RESYNC(*bmc))
1522 rv = 1;
1523 else if (NEEDED(*bmc)) {
1524 rv = 1;
1525 if (!degraded) {
1526 *bmc |= RESYNC_MASK;
1527 *bmc &= ~NEEDED_MASK;
1528 }
1529 }
1530 }
1531 spin_unlock_irq(&bitmap->lock);
1532 bitmap->allclean = 0;
1533 return rv;
1534}
1535
1536int bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks,
1537 int degraded)
1538{
1539
1540
1541
1542
1543
1544
1545
1546 int rv = 0;
1547 sector_t blocks1;
1548
1549 *blocks = 0;
1550 while (*blocks < (PAGE_SIZE>>9)) {
1551 rv |= __bitmap_start_sync(bitmap, offset,
1552 &blocks1, degraded);
1553 offset += blocks1;
1554 *blocks += blocks1;
1555 }
1556 return rv;
1557}
1558EXPORT_SYMBOL(bitmap_start_sync);
1559
1560void bitmap_end_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks, int aborted)
1561{
1562 bitmap_counter_t *bmc;
1563 unsigned long flags;
1564
1565 if (bitmap == NULL) {
1566 *blocks = 1024;
1567 return;
1568 }
1569 spin_lock_irqsave(&bitmap->lock, flags);
1570 bmc = bitmap_get_counter(bitmap, offset, blocks, 0);
1571 if (bmc == NULL)
1572 goto unlock;
1573
1574 if (RESYNC(*bmc)) {
1575 *bmc &= ~RESYNC_MASK;
1576
1577 if (!NEEDED(*bmc) && aborted)
1578 *bmc |= NEEDED_MASK;
1579 else {
1580 if (*bmc <= 2)
1581 set_page_attr(bitmap,
1582 filemap_get_page(bitmap, offset >> CHUNK_BLOCK_SHIFT(bitmap)),
1583 BITMAP_PAGE_CLEAN);
1584 }
1585 }
1586 unlock:
1587 spin_unlock_irqrestore(&bitmap->lock, flags);
1588 bitmap->allclean = 0;
1589}
1590EXPORT_SYMBOL(bitmap_end_sync);
1591
1592void bitmap_close_sync(struct bitmap *bitmap)
1593{
1594
1595
1596
1597
1598 sector_t sector = 0;
1599 sector_t blocks;
1600 if (!bitmap)
1601 return;
1602 while (sector < bitmap->mddev->resync_max_sectors) {
1603 bitmap_end_sync(bitmap, sector, &blocks, 0);
1604 sector += blocks;
1605 }
1606}
1607EXPORT_SYMBOL(bitmap_close_sync);
1608
1609void bitmap_cond_end_sync(struct bitmap *bitmap, sector_t sector)
1610{
1611 sector_t s = 0;
1612 sector_t blocks;
1613
1614 if (!bitmap)
1615 return;
1616 if (sector == 0) {
1617 bitmap->last_end_sync = jiffies;
1618 return;
1619 }
1620 if (time_before(jiffies, (bitmap->last_end_sync
1621 + bitmap->mddev->bitmap_info.daemon_sleep)))
1622 return;
1623 wait_event(bitmap->mddev->recovery_wait,
1624 atomic_read(&bitmap->mddev->recovery_active) == 0);
1625
1626 bitmap->mddev->curr_resync_completed = sector;
1627 set_bit(MD_CHANGE_CLEAN, &bitmap->mddev->flags);
1628 sector &= ~((1ULL << CHUNK_BLOCK_SHIFT(bitmap)) - 1);
1629 s = 0;
1630 while (s < sector && s < bitmap->mddev->resync_max_sectors) {
1631 bitmap_end_sync(bitmap, s, &blocks, 0);
1632 s += blocks;
1633 }
1634 bitmap->last_end_sync = jiffies;
1635 sysfs_notify(&bitmap->mddev->kobj, NULL, "sync_completed");
1636}
1637EXPORT_SYMBOL(bitmap_cond_end_sync);
1638
1639static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed)
1640{
1641
1642
1643
1644
1645
1646 sector_t secs;
1647 bitmap_counter_t *bmc;
1648 spin_lock_irq(&bitmap->lock);
1649 bmc = bitmap_get_counter(bitmap, offset, &secs, 1);
1650 if (!bmc) {
1651 spin_unlock_irq(&bitmap->lock);
1652 return;
1653 }
1654 if (!*bmc) {
1655 struct page *page;
1656 *bmc = 1 | (needed ? NEEDED_MASK : 0);
1657 bitmap_count_page(bitmap, offset, 1);
1658 page = filemap_get_page(bitmap, offset >> CHUNK_BLOCK_SHIFT(bitmap));
1659 set_page_attr(bitmap, page, BITMAP_PAGE_CLEAN);
1660 }
1661 spin_unlock_irq(&bitmap->lock);
1662 bitmap->allclean = 0;
1663}
1664
1665
1666void bitmap_dirty_bits(struct bitmap *bitmap, unsigned long s, unsigned long e)
1667{
1668 unsigned long chunk;
1669
1670 for (chunk = s; chunk <= e; chunk++) {
1671 sector_t sec = (sector_t)chunk << CHUNK_BLOCK_SHIFT(bitmap);
1672 bitmap_set_memory_bits(bitmap, sec, 1);
1673 bitmap_file_set_bit(bitmap, sec);
1674 if (sec < bitmap->mddev->recovery_cp)
1675
1676
1677
1678
1679 bitmap->mddev->recovery_cp = sec;
1680 }
1681}
1682
1683
1684
1685
1686void bitmap_flush(mddev_t *mddev)
1687{
1688 struct bitmap *bitmap = mddev->bitmap;
1689 long sleep;
1690
1691 if (!bitmap)
1692 return;
1693
1694
1695
1696
1697 sleep = mddev->bitmap_info.daemon_sleep * 2;
1698 bitmap->daemon_lastrun -= sleep;
1699 bitmap_daemon_work(mddev);
1700 bitmap->daemon_lastrun -= sleep;
1701 bitmap_daemon_work(mddev);
1702 bitmap->daemon_lastrun -= sleep;
1703 bitmap_daemon_work(mddev);
1704 bitmap_update_sb(bitmap);
1705}
1706
1707
1708
1709
1710static void bitmap_free(struct bitmap *bitmap)
1711{
1712 unsigned long k, pages;
1713 struct bitmap_page *bp;
1714
1715 if (!bitmap)
1716 return;
1717
1718
1719 bitmap_file_put(bitmap);
1720
1721 bp = bitmap->bp;
1722 pages = bitmap->pages;
1723
1724
1725
1726 if (bp)
1727 for (k = 0; k < pages; k++)
1728 if (bp[k].map && !bp[k].hijacked)
1729 kfree(bp[k].map);
1730 kfree(bp);
1731 kfree(bitmap);
1732}
1733
1734void bitmap_destroy(mddev_t *mddev)
1735{
1736 struct bitmap *bitmap = mddev->bitmap;
1737
1738 if (!bitmap)
1739 return;
1740
1741 mutex_lock(&mddev->bitmap_info.mutex);
1742 mddev->bitmap = NULL;
1743 mutex_unlock(&mddev->bitmap_info.mutex);
1744 if (mddev->thread)
1745 mddev->thread->timeout = MAX_SCHEDULE_TIMEOUT;
1746
1747 if (bitmap->sysfs_can_clear)
1748 sysfs_put(bitmap->sysfs_can_clear);
1749
1750 bitmap_free(bitmap);
1751}
1752
1753
1754
1755
1756
1757int bitmap_create(mddev_t *mddev)
1758{
1759 struct bitmap *bitmap;
1760 sector_t blocks = mddev->resync_max_sectors;
1761 unsigned long chunks;
1762 unsigned long pages;
1763 struct file *file = mddev->bitmap_info.file;
1764 int err;
1765 struct sysfs_dirent *bm = NULL;
1766
1767 BUILD_BUG_ON(sizeof(bitmap_super_t) != 256);
1768
1769 if (!file
1770 && !mddev->bitmap_info.offset
1771 && !mddev->bitmap_info.log)
1772 return 0;
1773
1774 BUG_ON(file && mddev->bitmap_info.offset);
1775 BUG_ON(mddev->bitmap_info.offset && mddev->bitmap_info.log);
1776
1777 bitmap = kzalloc(sizeof(*bitmap), GFP_KERNEL);
1778 if (!bitmap)
1779 return -ENOMEM;
1780
1781 spin_lock_init(&bitmap->lock);
1782 atomic_set(&bitmap->pending_writes, 0);
1783 init_waitqueue_head(&bitmap->write_wait);
1784 init_waitqueue_head(&bitmap->overflow_wait);
1785 init_waitqueue_head(&bitmap->behind_wait);
1786
1787 bitmap->mddev = mddev;
1788
1789 if (mddev->kobj.sd)
1790 bm = sysfs_get_dirent(mddev->kobj.sd, NULL, "bitmap");
1791 if (bm) {
1792 bitmap->sysfs_can_clear = sysfs_get_dirent(bm, NULL, "can_clear");
1793 sysfs_put(bm);
1794 } else
1795 bitmap->sysfs_can_clear = NULL;
1796
1797 bitmap->file = file;
1798 if (file) {
1799 get_file(file);
1800
1801
1802
1803
1804 vfs_fsync(file, 1);
1805 }
1806
1807 if (!mddev->bitmap_info.external) {
1808
1809
1810
1811
1812 if (test_and_clear_bit(MD_ARRAY_FIRST_USE, &mddev->flags))
1813 err = bitmap_new_disk_sb(bitmap);
1814 else
1815 err = bitmap_read_sb(bitmap);
1816 } else {
1817 err = 0;
1818 if (mddev->bitmap_info.chunksize == 0 ||
1819 mddev->bitmap_info.daemon_sleep == 0)
1820
1821
1822 err = -EINVAL;
1823 }
1824 if (err)
1825 goto error;
1826
1827 bitmap->daemon_lastrun = jiffies;
1828 bitmap->chunkshift = ffz(~mddev->bitmap_info.chunksize);
1829
1830
1831 chunks = (blocks + CHUNK_BLOCK_RATIO(bitmap) - 1) >>
1832 CHUNK_BLOCK_SHIFT(bitmap);
1833 pages = (chunks + PAGE_COUNTER_RATIO - 1) / PAGE_COUNTER_RATIO;
1834
1835 BUG_ON(!pages);
1836
1837 bitmap->chunks = chunks;
1838 bitmap->pages = pages;
1839 bitmap->missing_pages = pages;
1840
1841#ifdef INJECT_FATAL_FAULT_1
1842 bitmap->bp = NULL;
1843#else
1844 bitmap->bp = kzalloc(pages * sizeof(*bitmap->bp), GFP_KERNEL);
1845#endif
1846 err = -ENOMEM;
1847 if (!bitmap->bp)
1848 goto error;
1849
1850 printk(KERN_INFO "created bitmap (%lu pages) for device %s\n",
1851 pages, bmname(bitmap));
1852
1853 mddev->bitmap = bitmap;
1854
1855
1856 return (bitmap->flags & BITMAP_WRITE_ERROR) ? -EIO : 0;
1857
1858 error:
1859 bitmap_free(bitmap);
1860 return err;
1861}
1862
1863int bitmap_load(mddev_t *mddev)
1864{
1865 int err = 0;
1866 sector_t sector = 0;
1867 struct bitmap *bitmap = mddev->bitmap;
1868
1869 if (!bitmap)
1870 goto out;
1871
1872
1873
1874
1875
1876
1877 while (sector < mddev->resync_max_sectors) {
1878 sector_t blocks;
1879 bitmap_start_sync(bitmap, sector, &blocks, 0);
1880 sector += blocks;
1881 }
1882 bitmap_close_sync(bitmap);
1883
1884 if (mddev->bitmap_info.log) {
1885 unsigned long i;
1886 struct dm_dirty_log *log = mddev->bitmap_info.log;
1887 for (i = 0; i < bitmap->chunks; i++)
1888 if (!log->type->in_sync(log, i, 1))
1889 bitmap_set_memory_bits(bitmap,
1890 (sector_t)i << CHUNK_BLOCK_SHIFT(bitmap),
1891 1);
1892 } else {
1893 sector_t start = 0;
1894 if (mddev->degraded == 0
1895 || bitmap->events_cleared == mddev->events)
1896
1897
1898 start = mddev->recovery_cp;
1899
1900 err = bitmap_init_from_disk(bitmap, start);
1901 }
1902 if (err)
1903 goto out;
1904
1905 mddev->thread->timeout = mddev->bitmap_info.daemon_sleep;
1906 md_wakeup_thread(mddev->thread);
1907
1908 bitmap_update_sb(bitmap);
1909
1910 if (bitmap->flags & BITMAP_WRITE_ERROR)
1911 err = -EIO;
1912out:
1913 return err;
1914}
1915EXPORT_SYMBOL_GPL(bitmap_load);
1916
1917static ssize_t
1918location_show(mddev_t *mddev, char *page)
1919{
1920 ssize_t len;
1921 if (mddev->bitmap_info.file)
1922 len = sprintf(page, "file");
1923 else if (mddev->bitmap_info.offset)
1924 len = sprintf(page, "%+lld", (long long)mddev->bitmap_info.offset);
1925 else
1926 len = sprintf(page, "none");
1927 len += sprintf(page+len, "\n");
1928 return len;
1929}
1930
1931static ssize_t
1932location_store(mddev_t *mddev, const char *buf, size_t len)
1933{
1934
1935 if (mddev->pers) {
1936 if (!mddev->pers->quiesce)
1937 return -EBUSY;
1938 if (mddev->recovery || mddev->sync_thread)
1939 return -EBUSY;
1940 }
1941
1942 if (mddev->bitmap || mddev->bitmap_info.file ||
1943 mddev->bitmap_info.offset) {
1944
1945 if (strncmp(buf, "none", 4) != 0)
1946 return -EBUSY;
1947 if (mddev->pers) {
1948 mddev->pers->quiesce(mddev, 1);
1949 bitmap_destroy(mddev);
1950 mddev->pers->quiesce(mddev, 0);
1951 }
1952 mddev->bitmap_info.offset = 0;
1953 if (mddev->bitmap_info.file) {
1954 struct file *f = mddev->bitmap_info.file;
1955 mddev->bitmap_info.file = NULL;
1956 restore_bitmap_write_access(f);
1957 fput(f);
1958 }
1959 } else {
1960
1961 long long offset;
1962 if (strncmp(buf, "none", 4) == 0)
1963 ;
1964 else if (strncmp(buf, "file:", 5) == 0) {
1965
1966 return -EINVAL;
1967 } else {
1968 int rv;
1969 if (buf[0] == '+')
1970 rv = strict_strtoll(buf+1, 10, &offset);
1971 else
1972 rv = strict_strtoll(buf, 10, &offset);
1973 if (rv)
1974 return rv;
1975 if (offset == 0)
1976 return -EINVAL;
1977 if (mddev->bitmap_info.external == 0 &&
1978 mddev->major_version == 0 &&
1979 offset != mddev->bitmap_info.default_offset)
1980 return -EINVAL;
1981 mddev->bitmap_info.offset = offset;
1982 if (mddev->pers) {
1983 mddev->pers->quiesce(mddev, 1);
1984 rv = bitmap_create(mddev);
1985 if (rv) {
1986 bitmap_destroy(mddev);
1987 mddev->bitmap_info.offset = 0;
1988 }
1989 mddev->pers->quiesce(mddev, 0);
1990 if (rv)
1991 return rv;
1992 }
1993 }
1994 }
1995 if (!mddev->external) {
1996
1997
1998
1999 set_bit(MD_CHANGE_DEVS, &mddev->flags);
2000 md_wakeup_thread(mddev->thread);
2001 }
2002 return len;
2003}
2004
2005static struct md_sysfs_entry bitmap_location =
2006__ATTR(location, S_IRUGO|S_IWUSR, location_show, location_store);
2007
2008static ssize_t
2009timeout_show(mddev_t *mddev, char *page)
2010{
2011 ssize_t len;
2012 unsigned long secs = mddev->bitmap_info.daemon_sleep / HZ;
2013 unsigned long jifs = mddev->bitmap_info.daemon_sleep % HZ;
2014
2015 len = sprintf(page, "%lu", secs);
2016 if (jifs)
2017 len += sprintf(page+len, ".%03u", jiffies_to_msecs(jifs));
2018 len += sprintf(page+len, "\n");
2019 return len;
2020}
2021
2022static ssize_t
2023timeout_store(mddev_t *mddev, const char *buf, size_t len)
2024{
2025
2026 unsigned long timeout;
2027 int rv = strict_strtoul_scaled(buf, &timeout, 4);
2028 if (rv)
2029 return rv;
2030
2031
2032 if (timeout >= LONG_MAX / HZ)
2033 return -EINVAL;
2034
2035 timeout = timeout * HZ / 10000;
2036
2037 if (timeout >= MAX_SCHEDULE_TIMEOUT)
2038 timeout = MAX_SCHEDULE_TIMEOUT-1;
2039 if (timeout < 1)
2040 timeout = 1;
2041 mddev->bitmap_info.daemon_sleep = timeout;
2042 if (mddev->thread) {
2043
2044
2045
2046
2047 if (mddev->thread->timeout < MAX_SCHEDULE_TIMEOUT) {
2048 mddev->thread->timeout = timeout;
2049 md_wakeup_thread(mddev->thread);
2050 }
2051 }
2052 return len;
2053}
2054
2055static struct md_sysfs_entry bitmap_timeout =
2056__ATTR(time_base, S_IRUGO|S_IWUSR, timeout_show, timeout_store);
2057
2058static ssize_t
2059backlog_show(mddev_t *mddev, char *page)
2060{
2061 return sprintf(page, "%lu\n", mddev->bitmap_info.max_write_behind);
2062}
2063
2064static ssize_t
2065backlog_store(mddev_t *mddev, const char *buf, size_t len)
2066{
2067 unsigned long backlog;
2068 int rv = strict_strtoul(buf, 10, &backlog);
2069 if (rv)
2070 return rv;
2071 if (backlog > COUNTER_MAX)
2072 return -EINVAL;
2073 mddev->bitmap_info.max_write_behind = backlog;
2074 return len;
2075}
2076
2077static struct md_sysfs_entry bitmap_backlog =
2078__ATTR(backlog, S_IRUGO|S_IWUSR, backlog_show, backlog_store);
2079
2080static ssize_t
2081chunksize_show(mddev_t *mddev, char *page)
2082{
2083 return sprintf(page, "%lu\n", mddev->bitmap_info.chunksize);
2084}
2085
2086static ssize_t
2087chunksize_store(mddev_t *mddev, const char *buf, size_t len)
2088{
2089
2090 int rv;
2091 unsigned long csize;
2092 if (mddev->bitmap)
2093 return -EBUSY;
2094 rv = strict_strtoul(buf, 10, &csize);
2095 if (rv)
2096 return rv;
2097 if (csize < 512 ||
2098 !is_power_of_2(csize))
2099 return -EINVAL;
2100 mddev->bitmap_info.chunksize = csize;
2101 return len;
2102}
2103
2104static struct md_sysfs_entry bitmap_chunksize =
2105__ATTR(chunksize, S_IRUGO|S_IWUSR, chunksize_show, chunksize_store);
2106
2107static ssize_t metadata_show(mddev_t *mddev, char *page)
2108{
2109 return sprintf(page, "%s\n", (mddev->bitmap_info.external
2110 ? "external" : "internal"));
2111}
2112
2113static ssize_t metadata_store(mddev_t *mddev, const char *buf, size_t len)
2114{
2115 if (mddev->bitmap ||
2116 mddev->bitmap_info.file ||
2117 mddev->bitmap_info.offset)
2118 return -EBUSY;
2119 if (strncmp(buf, "external", 8) == 0)
2120 mddev->bitmap_info.external = 1;
2121 else if (strncmp(buf, "internal", 8) == 0)
2122 mddev->bitmap_info.external = 0;
2123 else
2124 return -EINVAL;
2125 return len;
2126}
2127
2128static struct md_sysfs_entry bitmap_metadata =
2129__ATTR(metadata, S_IRUGO|S_IWUSR, metadata_show, metadata_store);
2130
2131static ssize_t can_clear_show(mddev_t *mddev, char *page)
2132{
2133 int len;
2134 if (mddev->bitmap)
2135 len = sprintf(page, "%s\n", (mddev->bitmap->need_sync ?
2136 "false" : "true"));
2137 else
2138 len = sprintf(page, "\n");
2139 return len;
2140}
2141
2142static ssize_t can_clear_store(mddev_t *mddev, const char *buf, size_t len)
2143{
2144 if (mddev->bitmap == NULL)
2145 return -ENOENT;
2146 if (strncmp(buf, "false", 5) == 0)
2147 mddev->bitmap->need_sync = 1;
2148 else if (strncmp(buf, "true", 4) == 0) {
2149 if (mddev->degraded)
2150 return -EBUSY;
2151 mddev->bitmap->need_sync = 0;
2152 } else
2153 return -EINVAL;
2154 return len;
2155}
2156
2157static struct md_sysfs_entry bitmap_can_clear =
2158__ATTR(can_clear, S_IRUGO|S_IWUSR, can_clear_show, can_clear_store);
2159
2160static ssize_t
2161behind_writes_used_show(mddev_t *mddev, char *page)
2162{
2163 if (mddev->bitmap == NULL)
2164 return sprintf(page, "0\n");
2165 return sprintf(page, "%lu\n",
2166 mddev->bitmap->behind_writes_used);
2167}
2168
2169static ssize_t
2170behind_writes_used_reset(mddev_t *mddev, const char *buf, size_t len)
2171{
2172 if (mddev->bitmap)
2173 mddev->bitmap->behind_writes_used = 0;
2174 return len;
2175}
2176
2177static struct md_sysfs_entry max_backlog_used =
2178__ATTR(max_backlog_used, S_IRUGO | S_IWUSR,
2179 behind_writes_used_show, behind_writes_used_reset);
2180
2181static struct attribute *md_bitmap_attrs[] = {
2182 &bitmap_location.attr,
2183 &bitmap_timeout.attr,
2184 &bitmap_backlog.attr,
2185 &bitmap_chunksize.attr,
2186 &bitmap_metadata.attr,
2187 &bitmap_can_clear.attr,
2188 &max_backlog_used.attr,
2189 NULL
2190};
2191struct attribute_group md_bitmap_group = {
2192 .name = "bitmap",
2193 .attrs = md_bitmap_attrs,
2194};
2195
2196