1
2
3
4
5
6
7
8
9
10
11
12
13
14#include <linux/slab.h>
15#include <linux/crc32c.h>
16#include <linux/drbd.h>
17#include <linux/drbd_limits.h>
18#include "drbd_int.h"
19
20
21enum al_transaction_types {
22 AL_TR_UPDATE = 0,
23 AL_TR_INITIALIZED = 0xffff
24};
25
26struct __packed al_transaction_on_disk {
27
28 __be32 magic;
29
30
31
32 __be32 tr_number;
33
34
35 __be32 crc32c;
36
37
38
39
40 __be16 transaction_type;
41
42
43
44
45
46 __be16 n_updates;
47
48
49
50
51 __be16 context_size;
52
53
54 __be16 context_start_slot_nr;
55
56
57
58
59 __be32 __reserved[4];
60
61
62
63
64
65
66
67
68
69
70 __be16 update_slot_nr[AL_UPDATES_PER_TRANSACTION];
71
72
73
74 __be32 update_extent_nr[AL_UPDATES_PER_TRANSACTION];
75
76
77
78
79 __be32 context[AL_CONTEXT_PER_TRANSACTION];
80};
81
82void *drbd_md_get_buffer(struct drbd_device *device, const char *intent)
83{
84 int r;
85
86 wait_event(device->misc_wait,
87 (r = atomic_cmpxchg(&device->md_io.in_use, 0, 1)) == 0 ||
88 device->state.disk <= D_FAILED);
89
90 if (r)
91 return NULL;
92
93 device->md_io.current_use = intent;
94 device->md_io.start_jif = jiffies;
95 device->md_io.submit_jif = device->md_io.start_jif - 1;
96 return page_address(device->md_io.page);
97}
98
99void drbd_md_put_buffer(struct drbd_device *device)
100{
101 if (atomic_dec_and_test(&device->md_io.in_use))
102 wake_up(&device->misc_wait);
103}
104
105void wait_until_done_or_force_detached(struct drbd_device *device, struct drbd_backing_dev *bdev,
106 unsigned int *done)
107{
108 long dt;
109
110 rcu_read_lock();
111 dt = rcu_dereference(bdev->disk_conf)->disk_timeout;
112 rcu_read_unlock();
113 dt = dt * HZ / 10;
114 if (dt == 0)
115 dt = MAX_SCHEDULE_TIMEOUT;
116
117 dt = wait_event_timeout(device->misc_wait,
118 *done || test_bit(FORCE_DETACH, &device->flags), dt);
119 if (dt == 0) {
120 drbd_err(device, "meta-data IO operation timed out\n");
121 drbd_chk_io_error(device, 1, DRBD_FORCE_DETACH);
122 }
123}
124
125static int _drbd_md_sync_page_io(struct drbd_device *device,
126 struct drbd_backing_dev *bdev,
127 sector_t sector, int op)
128{
129 struct bio *bio;
130
131 const int size = 4096;
132 int err, op_flags = 0;
133
134 device->md_io.done = 0;
135 device->md_io.error = -ENODEV;
136
137 if ((op == REQ_OP_WRITE) && !test_bit(MD_NO_FUA, &device->flags))
138 op_flags |= REQ_FUA | REQ_PREFLUSH;
139 op_flags |= REQ_SYNC;
140
141 bio = bio_alloc_bioset(bdev->md_bdev, 1, op | op_flags, GFP_NOIO,
142 &drbd_md_io_bio_set);
143 bio->bi_iter.bi_sector = sector;
144 err = -EIO;
145 if (bio_add_page(bio, device->md_io.page, size, 0) != size)
146 goto out;
147 bio->bi_private = device;
148 bio->bi_end_io = drbd_md_endio;
149
150 if (op != REQ_OP_WRITE && device->state.disk == D_DISKLESS && device->ldev == NULL)
151
152 ;
153 else if (!get_ldev_if_state(device, D_ATTACHING)) {
154
155 drbd_err(device, "ASSERT FAILED: get_ldev_if_state() == 1 in _drbd_md_sync_page_io()\n");
156 err = -ENODEV;
157 goto out;
158 }
159
160 bio_get(bio);
161 atomic_inc(&device->md_io.in_use);
162 device->md_io.submit_jif = jiffies;
163 if (drbd_insert_fault(device, (op == REQ_OP_WRITE) ? DRBD_FAULT_MD_WR : DRBD_FAULT_MD_RD))
164 bio_io_error(bio);
165 else
166 submit_bio(bio);
167 wait_until_done_or_force_detached(device, bdev, &device->md_io.done);
168 if (!bio->bi_status)
169 err = device->md_io.error;
170
171 out:
172 bio_put(bio);
173 return err;
174}
175
176int drbd_md_sync_page_io(struct drbd_device *device, struct drbd_backing_dev *bdev,
177 sector_t sector, int op)
178{
179 int err;
180 D_ASSERT(device, atomic_read(&device->md_io.in_use) == 1);
181
182 BUG_ON(!bdev->md_bdev);
183
184 dynamic_drbd_dbg(device, "meta_data io: %s [%d]:%s(,%llus,%s) %pS\n",
185 current->comm, current->pid, __func__,
186 (unsigned long long)sector, (op == REQ_OP_WRITE) ? "WRITE" : "READ",
187 (void*)_RET_IP_ );
188
189 if (sector < drbd_md_first_sector(bdev) ||
190 sector + 7 > drbd_md_last_sector(bdev))
191 drbd_alert(device, "%s [%d]:%s(,%llus,%s) out of range md access!\n",
192 current->comm, current->pid, __func__,
193 (unsigned long long)sector,
194 (op == REQ_OP_WRITE) ? "WRITE" : "READ");
195
196 err = _drbd_md_sync_page_io(device, bdev, sector, op);
197 if (err) {
198 drbd_err(device, "drbd_md_sync_page_io(,%llus,%s) failed with error %d\n",
199 (unsigned long long)sector,
200 (op == REQ_OP_WRITE) ? "WRITE" : "READ", err);
201 }
202 return err;
203}
204
205static struct bm_extent *find_active_resync_extent(struct drbd_device *device, unsigned int enr)
206{
207 struct lc_element *tmp;
208 tmp = lc_find(device->resync, enr/AL_EXT_PER_BM_SECT);
209 if (unlikely(tmp != NULL)) {
210 struct bm_extent *bm_ext = lc_entry(tmp, struct bm_extent, lce);
211 if (test_bit(BME_NO_WRITES, &bm_ext->flags))
212 return bm_ext;
213 }
214 return NULL;
215}
216
217static struct lc_element *_al_get(struct drbd_device *device, unsigned int enr, bool nonblock)
218{
219 struct lc_element *al_ext;
220 struct bm_extent *bm_ext;
221 int wake;
222
223 spin_lock_irq(&device->al_lock);
224 bm_ext = find_active_resync_extent(device, enr);
225 if (bm_ext) {
226 wake = !test_and_set_bit(BME_PRIORITY, &bm_ext->flags);
227 spin_unlock_irq(&device->al_lock);
228 if (wake)
229 wake_up(&device->al_wait);
230 return NULL;
231 }
232 if (nonblock)
233 al_ext = lc_try_get(device->act_log, enr);
234 else
235 al_ext = lc_get(device->act_log, enr);
236 spin_unlock_irq(&device->al_lock);
237 return al_ext;
238}
239
240bool drbd_al_begin_io_fastpath(struct drbd_device *device, struct drbd_interval *i)
241{
242
243
244 unsigned first = i->sector >> (AL_EXTENT_SHIFT-9);
245 unsigned last = i->size == 0 ? first : (i->sector + (i->size >> 9) - 1) >> (AL_EXTENT_SHIFT-9);
246
247 D_ASSERT(device, first <= last);
248 D_ASSERT(device, atomic_read(&device->local_cnt) > 0);
249
250
251 if (first != last)
252 return false;
253
254 return _al_get(device, first, true);
255}
256
257bool drbd_al_begin_io_prepare(struct drbd_device *device, struct drbd_interval *i)
258{
259
260
261 unsigned first = i->sector >> (AL_EXTENT_SHIFT-9);
262 unsigned last = i->size == 0 ? first : (i->sector + (i->size >> 9) - 1) >> (AL_EXTENT_SHIFT-9);
263 unsigned enr;
264 bool need_transaction = false;
265
266 D_ASSERT(device, first <= last);
267 D_ASSERT(device, atomic_read(&device->local_cnt) > 0);
268
269 for (enr = first; enr <= last; enr++) {
270 struct lc_element *al_ext;
271 wait_event(device->al_wait,
272 (al_ext = _al_get(device, enr, false)) != NULL);
273 if (al_ext->lc_number != enr)
274 need_transaction = true;
275 }
276 return need_transaction;
277}
278
279#if (PAGE_SHIFT + 3) < (AL_EXTENT_SHIFT - BM_BLOCK_SHIFT)
280
281
282
283
284
285# error FIXME
286#endif
287
288static unsigned int al_extent_to_bm_page(unsigned int al_enr)
289{
290 return al_enr >>
291
292 ((PAGE_SHIFT + 3) -
293
294 (AL_EXTENT_SHIFT - BM_BLOCK_SHIFT));
295}
296
297static sector_t al_tr_number_to_on_disk_sector(struct drbd_device *device)
298{
299 const unsigned int stripes = device->ldev->md.al_stripes;
300 const unsigned int stripe_size_4kB = device->ldev->md.al_stripe_size_4k;
301
302
303 unsigned int t = device->al_tr_number % (device->ldev->md.al_size_4k);
304
305
306 t = ((t % stripes) * stripe_size_4kB) + t/stripes;
307
308
309 t *= 8;
310
311
312 return device->ldev->md.md_offset + device->ldev->md.al_offset + t;
313}
314
315static int __al_write_transaction(struct drbd_device *device, struct al_transaction_on_disk *buffer)
316{
317 struct lc_element *e;
318 sector_t sector;
319 int i, mx;
320 unsigned extent_nr;
321 unsigned crc = 0;
322 int err = 0;
323
324 memset(buffer, 0, sizeof(*buffer));
325 buffer->magic = cpu_to_be32(DRBD_AL_MAGIC);
326 buffer->tr_number = cpu_to_be32(device->al_tr_number);
327
328 i = 0;
329
330 drbd_bm_reset_al_hints(device);
331
332
333
334
335
336 spin_lock_irq(&device->al_lock);
337 list_for_each_entry(e, &device->act_log->to_be_changed, list) {
338 if (i == AL_UPDATES_PER_TRANSACTION) {
339 i++;
340 break;
341 }
342 buffer->update_slot_nr[i] = cpu_to_be16(e->lc_index);
343 buffer->update_extent_nr[i] = cpu_to_be32(e->lc_new_number);
344 if (e->lc_number != LC_FREE)
345 drbd_bm_mark_for_writeout(device,
346 al_extent_to_bm_page(e->lc_number));
347 i++;
348 }
349 spin_unlock_irq(&device->al_lock);
350 BUG_ON(i > AL_UPDATES_PER_TRANSACTION);
351
352 buffer->n_updates = cpu_to_be16(i);
353 for ( ; i < AL_UPDATES_PER_TRANSACTION; i++) {
354 buffer->update_slot_nr[i] = cpu_to_be16(-1);
355 buffer->update_extent_nr[i] = cpu_to_be32(LC_FREE);
356 }
357
358 buffer->context_size = cpu_to_be16(device->act_log->nr_elements);
359 buffer->context_start_slot_nr = cpu_to_be16(device->al_tr_cycle);
360
361 mx = min_t(int, AL_CONTEXT_PER_TRANSACTION,
362 device->act_log->nr_elements - device->al_tr_cycle);
363 for (i = 0; i < mx; i++) {
364 unsigned idx = device->al_tr_cycle + i;
365 extent_nr = lc_element_by_index(device->act_log, idx)->lc_number;
366 buffer->context[i] = cpu_to_be32(extent_nr);
367 }
368 for (; i < AL_CONTEXT_PER_TRANSACTION; i++)
369 buffer->context[i] = cpu_to_be32(LC_FREE);
370
371 device->al_tr_cycle += AL_CONTEXT_PER_TRANSACTION;
372 if (device->al_tr_cycle >= device->act_log->nr_elements)
373 device->al_tr_cycle = 0;
374
375 sector = al_tr_number_to_on_disk_sector(device);
376
377 crc = crc32c(0, buffer, 4096);
378 buffer->crc32c = cpu_to_be32(crc);
379
380 if (drbd_bm_write_hinted(device))
381 err = -EIO;
382 else {
383 bool write_al_updates;
384 rcu_read_lock();
385 write_al_updates = rcu_dereference(device->ldev->disk_conf)->al_updates;
386 rcu_read_unlock();
387 if (write_al_updates) {
388 if (drbd_md_sync_page_io(device, device->ldev, sector, WRITE)) {
389 err = -EIO;
390 drbd_chk_io_error(device, 1, DRBD_META_IO_ERROR);
391 } else {
392 device->al_tr_number++;
393 device->al_writ_cnt++;
394 }
395 }
396 }
397
398 return err;
399}
400
401static int al_write_transaction(struct drbd_device *device)
402{
403 struct al_transaction_on_disk *buffer;
404 int err;
405
406 if (!get_ldev(device)) {
407 drbd_err(device, "disk is %s, cannot start al transaction\n",
408 drbd_disk_str(device->state.disk));
409 return -EIO;
410 }
411
412
413 if (device->state.disk < D_INCONSISTENT) {
414 drbd_err(device,
415 "disk is %s, cannot write al transaction\n",
416 drbd_disk_str(device->state.disk));
417 put_ldev(device);
418 return -EIO;
419 }
420
421
422 buffer = drbd_md_get_buffer(device, __func__);
423 if (!buffer) {
424 drbd_err(device, "disk failed while waiting for md_io buffer\n");
425 put_ldev(device);
426 return -ENODEV;
427 }
428
429 err = __al_write_transaction(device, buffer);
430
431 drbd_md_put_buffer(device);
432 put_ldev(device);
433
434 return err;
435}
436
437
438void drbd_al_begin_io_commit(struct drbd_device *device)
439{
440 bool locked = false;
441
442
443
444
445 wait_event(device->al_wait,
446 device->act_log->pending_changes == 0 ||
447 (locked = lc_try_lock_for_transaction(device->act_log)));
448
449 if (locked) {
450
451
452 if (device->act_log->pending_changes) {
453 bool write_al_updates;
454
455 rcu_read_lock();
456 write_al_updates = rcu_dereference(device->ldev->disk_conf)->al_updates;
457 rcu_read_unlock();
458
459 if (write_al_updates)
460 al_write_transaction(device);
461 spin_lock_irq(&device->al_lock);
462
463
464
465
466 lc_committed(device->act_log);
467 spin_unlock_irq(&device->al_lock);
468 }
469 lc_unlock(device->act_log);
470 wake_up(&device->al_wait);
471 }
472}
473
474
475
476
477void drbd_al_begin_io(struct drbd_device *device, struct drbd_interval *i)
478{
479 if (drbd_al_begin_io_prepare(device, i))
480 drbd_al_begin_io_commit(device);
481}
482
483int drbd_al_begin_io_nonblock(struct drbd_device *device, struct drbd_interval *i)
484{
485 struct lru_cache *al = device->act_log;
486
487
488 unsigned first = i->sector >> (AL_EXTENT_SHIFT-9);
489 unsigned last = i->size == 0 ? first : (i->sector + (i->size >> 9) - 1) >> (AL_EXTENT_SHIFT-9);
490 unsigned nr_al_extents;
491 unsigned available_update_slots;
492 unsigned enr;
493
494 D_ASSERT(device, first <= last);
495
496 nr_al_extents = 1 + last - first;
497 available_update_slots = min(al->nr_elements - al->used,
498 al->max_pending_changes - al->pending_changes);
499
500
501
502
503 if (available_update_slots < nr_al_extents) {
504
505
506
507
508
509
510
511
512 if (!al->pending_changes)
513 __set_bit(__LC_STARVING, &device->act_log->flags);
514 return -ENOBUFS;
515 }
516
517
518 for (enr = first; enr <= last; enr++) {
519 struct lc_element *tmp;
520 tmp = lc_find(device->resync, enr/AL_EXT_PER_BM_SECT);
521 if (unlikely(tmp != NULL)) {
522 struct bm_extent *bm_ext = lc_entry(tmp, struct bm_extent, lce);
523 if (test_bit(BME_NO_WRITES, &bm_ext->flags)) {
524 if (!test_and_set_bit(BME_PRIORITY, &bm_ext->flags))
525 return -EBUSY;
526 return -EWOULDBLOCK;
527 }
528 }
529 }
530
531
532
533
534 for (enr = first; enr <= last; enr++) {
535 struct lc_element *al_ext;
536 al_ext = lc_get_cumulative(device->act_log, enr);
537 if (!al_ext)
538 drbd_info(device, "LOGIC BUG for enr=%u\n", enr);
539 }
540 return 0;
541}
542
543void drbd_al_complete_io(struct drbd_device *device, struct drbd_interval *i)
544{
545
546
547 unsigned first = i->sector >> (AL_EXTENT_SHIFT-9);
548 unsigned last = i->size == 0 ? first : (i->sector + (i->size >> 9) - 1) >> (AL_EXTENT_SHIFT-9);
549 unsigned enr;
550 struct lc_element *extent;
551 unsigned long flags;
552
553 D_ASSERT(device, first <= last);
554 spin_lock_irqsave(&device->al_lock, flags);
555
556 for (enr = first; enr <= last; enr++) {
557 extent = lc_find(device->act_log, enr);
558 if (!extent) {
559 drbd_err(device, "al_complete_io() called on inactive extent %u\n", enr);
560 continue;
561 }
562 lc_put(device->act_log, extent);
563 }
564 spin_unlock_irqrestore(&device->al_lock, flags);
565 wake_up(&device->al_wait);
566}
567
568static int _try_lc_del(struct drbd_device *device, struct lc_element *al_ext)
569{
570 int rv;
571
572 spin_lock_irq(&device->al_lock);
573 rv = (al_ext->refcnt == 0);
574 if (likely(rv))
575 lc_del(device->act_log, al_ext);
576 spin_unlock_irq(&device->al_lock);
577
578 return rv;
579}
580
581
582
583
584
585
586
587
588
589
590void drbd_al_shrink(struct drbd_device *device)
591{
592 struct lc_element *al_ext;
593 int i;
594
595 D_ASSERT(device, test_bit(__LC_LOCKED, &device->act_log->flags));
596
597 for (i = 0; i < device->act_log->nr_elements; i++) {
598 al_ext = lc_element_by_index(device->act_log, i);
599 if (al_ext->lc_number == LC_FREE)
600 continue;
601 wait_event(device->al_wait, _try_lc_del(device, al_ext));
602 }
603
604 wake_up(&device->al_wait);
605}
606
607int drbd_al_initialize(struct drbd_device *device, void *buffer)
608{
609 struct al_transaction_on_disk *al = buffer;
610 struct drbd_md *md = &device->ldev->md;
611 int al_size_4k = md->al_stripes * md->al_stripe_size_4k;
612 int i;
613
614 __al_write_transaction(device, al);
615
616 spin_lock_irq(&device->al_lock);
617 lc_committed(device->act_log);
618 spin_unlock_irq(&device->al_lock);
619
620
621
622
623 for (i = 1; i < al_size_4k; i++) {
624 int err = __al_write_transaction(device, al);
625 if (err)
626 return err;
627 }
628 return 0;
629}
630
631static const char *drbd_change_sync_fname[] = {
632 [RECORD_RS_FAILED] = "drbd_rs_failed_io",
633 [SET_IN_SYNC] = "drbd_set_in_sync",
634 [SET_OUT_OF_SYNC] = "drbd_set_out_of_sync"
635};
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650static bool update_rs_extent(struct drbd_device *device,
651 unsigned int enr, int count,
652 enum update_sync_bits_mode mode)
653{
654 struct lc_element *e;
655
656 D_ASSERT(device, atomic_read(&device->local_cnt));
657
658
659
660
661
662
663
664
665 if (mode == SET_OUT_OF_SYNC)
666 e = lc_find(device->resync, enr);
667 else
668 e = lc_get(device->resync, enr);
669 if (e) {
670 struct bm_extent *ext = lc_entry(e, struct bm_extent, lce);
671 if (ext->lce.lc_number == enr) {
672 if (mode == SET_IN_SYNC)
673 ext->rs_left -= count;
674 else if (mode == SET_OUT_OF_SYNC)
675 ext->rs_left += count;
676 else
677 ext->rs_failed += count;
678 if (ext->rs_left < ext->rs_failed) {
679 drbd_warn(device, "BAD! enr=%u rs_left=%d "
680 "rs_failed=%d count=%d cstate=%s\n",
681 ext->lce.lc_number, ext->rs_left,
682 ext->rs_failed, count,
683 drbd_conn_str(device->state.conn));
684
685
686
687
688
689
690
691 ext->rs_left = drbd_bm_e_weight(device, enr);
692 }
693 } else {
694
695
696
697
698
699
700 int rs_left = drbd_bm_e_weight(device, enr);
701 if (ext->flags != 0) {
702 drbd_warn(device, "changing resync lce: %d[%u;%02lx]"
703 " -> %d[%u;00]\n",
704 ext->lce.lc_number, ext->rs_left,
705 ext->flags, enr, rs_left);
706 ext->flags = 0;
707 }
708 if (ext->rs_failed) {
709 drbd_warn(device, "Kicking resync_lru element enr=%u "
710 "out with rs_failed=%d\n",
711 ext->lce.lc_number, ext->rs_failed);
712 }
713 ext->rs_left = rs_left;
714 ext->rs_failed = (mode == RECORD_RS_FAILED) ? count : 0;
715
716
717 lc_committed(device->resync);
718 }
719 if (mode != SET_OUT_OF_SYNC)
720 lc_put(device->resync, &ext->lce);
721
722
723 if (ext->rs_left <= ext->rs_failed) {
724 ext->rs_failed = 0;
725 return true;
726 }
727 } else if (mode != SET_OUT_OF_SYNC) {
728
729 drbd_err(device, "lc_get() failed! locked=%d/%d flags=%lu\n",
730 device->resync_locked,
731 device->resync->nr_elements,
732 device->resync->flags);
733 }
734 return false;
735}
736
737void drbd_advance_rs_marks(struct drbd_device *device, unsigned long still_to_go)
738{
739 unsigned long now = jiffies;
740 unsigned long last = device->rs_mark_time[device->rs_last_mark];
741 int next = (device->rs_last_mark + 1) % DRBD_SYNC_MARKS;
742 if (time_after_eq(now, last + DRBD_SYNC_MARK_STEP)) {
743 if (device->rs_mark_left[device->rs_last_mark] != still_to_go &&
744 device->state.conn != C_PAUSED_SYNC_T &&
745 device->state.conn != C_PAUSED_SYNC_S) {
746 device->rs_mark_time[next] = now;
747 device->rs_mark_left[next] = still_to_go;
748 device->rs_last_mark = next;
749 }
750 }
751}
752
753
754static bool lazy_bitmap_update_due(struct drbd_device *device)
755{
756 return time_after(jiffies, device->rs_last_bcast + 2*HZ);
757}
758
759static void maybe_schedule_on_disk_bitmap_update(struct drbd_device *device, bool rs_done)
760{
761 if (rs_done) {
762 struct drbd_connection *connection = first_peer_device(device)->connection;
763 if (connection->agreed_pro_version <= 95 ||
764 is_sync_target_state(device->state.conn))
765 set_bit(RS_DONE, &device->flags);
766
767
768
769
770
771
772 } else if (!lazy_bitmap_update_due(device))
773 return;
774
775 drbd_device_post_work(device, RS_PROGRESS);
776}
777
778static int update_sync_bits(struct drbd_device *device,
779 unsigned long sbnr, unsigned long ebnr,
780 enum update_sync_bits_mode mode)
781{
782
783
784
785
786
787 unsigned long flags;
788 unsigned long count = 0;
789 unsigned int cleared = 0;
790 while (sbnr <= ebnr) {
791
792
793
794 unsigned long tbnr = min(ebnr, sbnr | BM_BLOCKS_PER_BM_EXT_MASK);
795 unsigned long c;
796
797 if (mode == RECORD_RS_FAILED)
798
799
800
801
802
803 c = drbd_bm_count_bits(device, sbnr, tbnr);
804 else if (mode == SET_IN_SYNC)
805 c = drbd_bm_clear_bits(device, sbnr, tbnr);
806 else
807 c = drbd_bm_set_bits(device, sbnr, tbnr);
808
809 if (c) {
810 spin_lock_irqsave(&device->al_lock, flags);
811 cleared += update_rs_extent(device, BM_BIT_TO_EXT(sbnr), c, mode);
812 spin_unlock_irqrestore(&device->al_lock, flags);
813 count += c;
814 }
815 sbnr = tbnr + 1;
816 }
817 if (count) {
818 if (mode == SET_IN_SYNC) {
819 unsigned long still_to_go = drbd_bm_total_weight(device);
820 bool rs_is_done = (still_to_go <= device->rs_failed);
821 drbd_advance_rs_marks(device, still_to_go);
822 if (cleared || rs_is_done)
823 maybe_schedule_on_disk_bitmap_update(device, rs_is_done);
824 } else if (mode == RECORD_RS_FAILED)
825 device->rs_failed += count;
826 wake_up(&device->al_wait);
827 }
828 return count;
829}
830
831static bool plausible_request_size(int size)
832{
833 return size > 0
834 && size <= DRBD_MAX_BATCH_BIO_SIZE
835 && IS_ALIGNED(size, 512);
836}
837
838
839
840
841
842
843
844
845int __drbd_change_sync(struct drbd_device *device, sector_t sector, int size,
846 enum update_sync_bits_mode mode)
847{
848
849 unsigned long sbnr, ebnr, lbnr;
850 unsigned long count = 0;
851 sector_t esector, nr_sectors;
852
853
854 if ((mode == SET_OUT_OF_SYNC) && size == 0)
855 return 0;
856
857 if (!plausible_request_size(size)) {
858 drbd_err(device, "%s: sector=%llus size=%d nonsense!\n",
859 drbd_change_sync_fname[mode],
860 (unsigned long long)sector, size);
861 return 0;
862 }
863
864 if (!get_ldev(device))
865 return 0;
866
867 nr_sectors = get_capacity(device->vdisk);
868 esector = sector + (size >> 9) - 1;
869
870 if (!expect(sector < nr_sectors))
871 goto out;
872 if (!expect(esector < nr_sectors))
873 esector = nr_sectors - 1;
874
875 lbnr = BM_SECT_TO_BIT(nr_sectors-1);
876
877 if (mode == SET_IN_SYNC) {
878
879
880 if (unlikely(esector < BM_SECT_PER_BIT-1))
881 goto out;
882 if (unlikely(esector == (nr_sectors-1)))
883 ebnr = lbnr;
884 else
885 ebnr = BM_SECT_TO_BIT(esector - (BM_SECT_PER_BIT-1));
886 sbnr = BM_SECT_TO_BIT(sector + BM_SECT_PER_BIT-1);
887 } else {
888
889
890 sbnr = BM_SECT_TO_BIT(sector);
891 ebnr = BM_SECT_TO_BIT(esector);
892 }
893
894 count = update_sync_bits(device, sbnr, ebnr, mode);
895out:
896 put_ldev(device);
897 return count;
898}
899
900static
901struct bm_extent *_bme_get(struct drbd_device *device, unsigned int enr)
902{
903 struct lc_element *e;
904 struct bm_extent *bm_ext;
905 int wakeup = 0;
906 unsigned long rs_flags;
907
908 spin_lock_irq(&device->al_lock);
909 if (device->resync_locked > device->resync->nr_elements/2) {
910 spin_unlock_irq(&device->al_lock);
911 return NULL;
912 }
913 e = lc_get(device->resync, enr);
914 bm_ext = e ? lc_entry(e, struct bm_extent, lce) : NULL;
915 if (bm_ext) {
916 if (bm_ext->lce.lc_number != enr) {
917 bm_ext->rs_left = drbd_bm_e_weight(device, enr);
918 bm_ext->rs_failed = 0;
919 lc_committed(device->resync);
920 wakeup = 1;
921 }
922 if (bm_ext->lce.refcnt == 1)
923 device->resync_locked++;
924 set_bit(BME_NO_WRITES, &bm_ext->flags);
925 }
926 rs_flags = device->resync->flags;
927 spin_unlock_irq(&device->al_lock);
928 if (wakeup)
929 wake_up(&device->al_wait);
930
931 if (!bm_ext) {
932 if (rs_flags & LC_STARVING)
933 drbd_warn(device, "Have to wait for element"
934 " (resync LRU too small?)\n");
935 BUG_ON(rs_flags & LC_LOCKED);
936 }
937
938 return bm_ext;
939}
940
941static int _is_in_al(struct drbd_device *device, unsigned int enr)
942{
943 int rv;
944
945 spin_lock_irq(&device->al_lock);
946 rv = lc_is_used(device->act_log, enr);
947 spin_unlock_irq(&device->al_lock);
948
949 return rv;
950}
951
952
953
954
955
956
957
958
959int drbd_rs_begin_io(struct drbd_device *device, sector_t sector)
960{
961 unsigned int enr = BM_SECT_TO_EXT(sector);
962 struct bm_extent *bm_ext;
963 int i, sig;
964 bool sa;
965
966retry:
967 sig = wait_event_interruptible(device->al_wait,
968 (bm_ext = _bme_get(device, enr)));
969 if (sig)
970 return -EINTR;
971
972 if (test_bit(BME_LOCKED, &bm_ext->flags))
973 return 0;
974
975
976 sa = drbd_rs_c_min_rate_throttle(device);
977
978 for (i = 0; i < AL_EXT_PER_BM_SECT; i++) {
979 sig = wait_event_interruptible(device->al_wait,
980 !_is_in_al(device, enr * AL_EXT_PER_BM_SECT + i) ||
981 (sa && test_bit(BME_PRIORITY, &bm_ext->flags)));
982
983 if (sig || (sa && test_bit(BME_PRIORITY, &bm_ext->flags))) {
984 spin_lock_irq(&device->al_lock);
985 if (lc_put(device->resync, &bm_ext->lce) == 0) {
986 bm_ext->flags = 0;
987 device->resync_locked--;
988 wake_up(&device->al_wait);
989 }
990 spin_unlock_irq(&device->al_lock);
991 if (sig)
992 return -EINTR;
993 if (schedule_timeout_interruptible(HZ/10))
994 return -EINTR;
995 goto retry;
996 }
997 }
998 set_bit(BME_LOCKED, &bm_ext->flags);
999 return 0;
1000}
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011int drbd_try_rs_begin_io(struct drbd_device *device, sector_t sector)
1012{
1013 unsigned int enr = BM_SECT_TO_EXT(sector);
1014 const unsigned int al_enr = enr*AL_EXT_PER_BM_SECT;
1015 struct lc_element *e;
1016 struct bm_extent *bm_ext;
1017 int i;
1018 bool throttle = drbd_rs_should_slow_down(device, sector, true);
1019
1020
1021
1022
1023
1024
1025 if (throttle && device->resync_wenr != enr)
1026 return -EAGAIN;
1027
1028 spin_lock_irq(&device->al_lock);
1029 if (device->resync_wenr != LC_FREE && device->resync_wenr != enr) {
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043 e = lc_find(device->resync, device->resync_wenr);
1044 bm_ext = e ? lc_entry(e, struct bm_extent, lce) : NULL;
1045 if (bm_ext) {
1046 D_ASSERT(device, !test_bit(BME_LOCKED, &bm_ext->flags));
1047 D_ASSERT(device, test_bit(BME_NO_WRITES, &bm_ext->flags));
1048 clear_bit(BME_NO_WRITES, &bm_ext->flags);
1049 device->resync_wenr = LC_FREE;
1050 if (lc_put(device->resync, &bm_ext->lce) == 0) {
1051 bm_ext->flags = 0;
1052 device->resync_locked--;
1053 }
1054 wake_up(&device->al_wait);
1055 } else {
1056 drbd_alert(device, "LOGIC BUG\n");
1057 }
1058 }
1059
1060 e = lc_try_get(device->resync, enr);
1061 bm_ext = e ? lc_entry(e, struct bm_extent, lce) : NULL;
1062 if (bm_ext) {
1063 if (test_bit(BME_LOCKED, &bm_ext->flags))
1064 goto proceed;
1065 if (!test_and_set_bit(BME_NO_WRITES, &bm_ext->flags)) {
1066 device->resync_locked++;
1067 } else {
1068
1069
1070
1071
1072 bm_ext->lce.refcnt--;
1073 D_ASSERT(device, bm_ext->lce.refcnt > 0);
1074 }
1075 goto check_al;
1076 } else {
1077
1078 if (device->resync_locked > device->resync->nr_elements-3)
1079 goto try_again;
1080
1081 e = lc_get(device->resync, enr);
1082 bm_ext = e ? lc_entry(e, struct bm_extent, lce) : NULL;
1083 if (!bm_ext) {
1084 const unsigned long rs_flags = device->resync->flags;
1085 if (rs_flags & LC_STARVING)
1086 drbd_warn(device, "Have to wait for element"
1087 " (resync LRU too small?)\n");
1088 BUG_ON(rs_flags & LC_LOCKED);
1089 goto try_again;
1090 }
1091 if (bm_ext->lce.lc_number != enr) {
1092 bm_ext->rs_left = drbd_bm_e_weight(device, enr);
1093 bm_ext->rs_failed = 0;
1094 lc_committed(device->resync);
1095 wake_up(&device->al_wait);
1096 D_ASSERT(device, test_bit(BME_LOCKED, &bm_ext->flags) == 0);
1097 }
1098 set_bit(BME_NO_WRITES, &bm_ext->flags);
1099 D_ASSERT(device, bm_ext->lce.refcnt == 1);
1100 device->resync_locked++;
1101 goto check_al;
1102 }
1103check_al:
1104 for (i = 0; i < AL_EXT_PER_BM_SECT; i++) {
1105 if (lc_is_used(device->act_log, al_enr+i))
1106 goto try_again;
1107 }
1108 set_bit(BME_LOCKED, &bm_ext->flags);
1109proceed:
1110 device->resync_wenr = LC_FREE;
1111 spin_unlock_irq(&device->al_lock);
1112 return 0;
1113
1114try_again:
1115 if (bm_ext) {
1116 if (throttle) {
1117 D_ASSERT(device, !test_bit(BME_LOCKED, &bm_ext->flags));
1118 D_ASSERT(device, test_bit(BME_NO_WRITES, &bm_ext->flags));
1119 clear_bit(BME_NO_WRITES, &bm_ext->flags);
1120 device->resync_wenr = LC_FREE;
1121 if (lc_put(device->resync, &bm_ext->lce) == 0) {
1122 bm_ext->flags = 0;
1123 device->resync_locked--;
1124 }
1125 wake_up(&device->al_wait);
1126 } else
1127 device->resync_wenr = enr;
1128 }
1129 spin_unlock_irq(&device->al_lock);
1130 return -EAGAIN;
1131}
1132
1133void drbd_rs_complete_io(struct drbd_device *device, sector_t sector)
1134{
1135 unsigned int enr = BM_SECT_TO_EXT(sector);
1136 struct lc_element *e;
1137 struct bm_extent *bm_ext;
1138 unsigned long flags;
1139
1140 spin_lock_irqsave(&device->al_lock, flags);
1141 e = lc_find(device->resync, enr);
1142 bm_ext = e ? lc_entry(e, struct bm_extent, lce) : NULL;
1143 if (!bm_ext) {
1144 spin_unlock_irqrestore(&device->al_lock, flags);
1145 if (__ratelimit(&drbd_ratelimit_state))
1146 drbd_err(device, "drbd_rs_complete_io() called, but extent not found\n");
1147 return;
1148 }
1149
1150 if (bm_ext->lce.refcnt == 0) {
1151 spin_unlock_irqrestore(&device->al_lock, flags);
1152 drbd_err(device, "drbd_rs_complete_io(,%llu [=%u]) called, "
1153 "but refcnt is 0!?\n",
1154 (unsigned long long)sector, enr);
1155 return;
1156 }
1157
1158 if (lc_put(device->resync, &bm_ext->lce) == 0) {
1159 bm_ext->flags = 0;
1160 device->resync_locked--;
1161 wake_up(&device->al_wait);
1162 }
1163
1164 spin_unlock_irqrestore(&device->al_lock, flags);
1165}
1166
1167
1168
1169
1170
1171void drbd_rs_cancel_all(struct drbd_device *device)
1172{
1173 spin_lock_irq(&device->al_lock);
1174
1175 if (get_ldev_if_state(device, D_FAILED)) {
1176 lc_reset(device->resync);
1177 put_ldev(device);
1178 }
1179 device->resync_locked = 0;
1180 device->resync_wenr = LC_FREE;
1181 spin_unlock_irq(&device->al_lock);
1182 wake_up(&device->al_wait);
1183}
1184
1185
1186
1187
1188
1189
1190
1191
1192int drbd_rs_del_all(struct drbd_device *device)
1193{
1194 struct lc_element *e;
1195 struct bm_extent *bm_ext;
1196 int i;
1197
1198 spin_lock_irq(&device->al_lock);
1199
1200 if (get_ldev_if_state(device, D_FAILED)) {
1201
1202 for (i = 0; i < device->resync->nr_elements; i++) {
1203 e = lc_element_by_index(device->resync, i);
1204 bm_ext = lc_entry(e, struct bm_extent, lce);
1205 if (bm_ext->lce.lc_number == LC_FREE)
1206 continue;
1207 if (bm_ext->lce.lc_number == device->resync_wenr) {
1208 drbd_info(device, "dropping %u in drbd_rs_del_all, apparently"
1209 " got 'synced' by application io\n",
1210 device->resync_wenr);
1211 D_ASSERT(device, !test_bit(BME_LOCKED, &bm_ext->flags));
1212 D_ASSERT(device, test_bit(BME_NO_WRITES, &bm_ext->flags));
1213 clear_bit(BME_NO_WRITES, &bm_ext->flags);
1214 device->resync_wenr = LC_FREE;
1215 lc_put(device->resync, &bm_ext->lce);
1216 }
1217 if (bm_ext->lce.refcnt != 0) {
1218 drbd_info(device, "Retrying drbd_rs_del_all() later. "
1219 "refcnt=%d\n", bm_ext->lce.refcnt);
1220 put_ldev(device);
1221 spin_unlock_irq(&device->al_lock);
1222 return -EAGAIN;
1223 }
1224 D_ASSERT(device, !test_bit(BME_LOCKED, &bm_ext->flags));
1225 D_ASSERT(device, !test_bit(BME_NO_WRITES, &bm_ext->flags));
1226 lc_del(device->resync, &bm_ext->lce);
1227 }
1228 D_ASSERT(device, device->resync->used == 0);
1229 put_ldev(device);
1230 }
1231 spin_unlock_irq(&device->al_lock);
1232 wake_up(&device->al_wait);
1233
1234 return 0;
1235}
1236